//---------------------------------------------------------------------------- // // File name: segLK.cpp // //---------------------------------------------------------------------------- // // Copyright (c) 2006, Rexee Inc. Doing fun things in Vision and AI, check us out www.rexee.com // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of Rexee Inc nor the // names of its contributors may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND AFFILIATED INSTITUTIONS ``AS IS'' AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED // WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE // DISCLAIMED. IN NO EVENT SHALL THE AUTHORS AND AFFILIATED INSTITUTIONS BE LIABLE FOR // ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES // (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND // ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // //---------------------------------------------------------------------------- // // Author: Gary R. Bradski // // Description: // This is a shell for reading from movies or camera, // displaying and interacting with the frames with slider and mouse. // // You must have the displayed "muti-window" active for keyboard commands to work. // Type 'h' to print a help file to consol. // This program reads in an image, does frame differencing with previous images and takes an hsv image. // All these are displayed in a "multi-window" that shows how you can put up many images, label them, and // view them. This file is the basis for creating and debugging vision algorithms (Stanley depended on it :-) ) // // You can draw rectangles in the upper left panel by clicking and dragging the left or right mouse button // (stay within the image panel or it will clip). The result will color or draw this interaction. You fill in // your own interaction such as filling pattern recognizers, reading statistics of regions etc. // // The routine "colorRectangle" is just there to show how to use image pointers to access data in a floating or // eight bit image. // // Text output can be to consol (just use "printf") OR it can be to the images themselves. Search on "UPDATE TEXT" below. // For text to image, you may have to adjust font size to fit, see "cvInitFont". // // Some simple OpenCV image processing is shown such as frame difference and conversion to HSV image. // // Note: Usually OpenCV color images from stills or movies are arrays of Blue,Green,Red pixels (not Red,Green,Blue) // So usually when converting (cvCvtColor) you *usually* use the form of the flags that begin with CV_BGR... // for example below: CV_BGR2GRAY to convert color to grayscale. // // Again: This file is to get you up and running to quickly display and interact with a panel of images, usually // you develop and algorithm and display the crutial processing stages on this (or instantialte multiple!) panels. // You can instantiate mouse interaction with each individual panel by setting panelCol and panelRow in the mouse // routine. // // The documentation for window interaction routines is in the "HighGUI" reference manual. // In the OpenCV\docs directory, open up the index.htm file, in separate browser tabs, open up: // CVCORE, CV Reference, Machine Learning Ref and HighGUI Reference to have OpenCV documentation handy. // // Notes: // // 11/4/06 Started // 1/10/07 Added selecting mouse rectangles and naming objects // //---------------------------------------------------------------------------- #ifdef _CH_ #pragma package #endif #ifndef _EiC #include "cv.h" #include "highgui.h" #include #include #include "cvx_multiwin.h" #endif int colorRectangle(IplImage *I, CvPoint pt_upperL, CvPoint pt_lowerR, int channel, float fputVal); // Some global variables: // // DEFINES #define FRAME_TIME_RESOLUTION 50 #define MAX_OBJECTS 100 // MOVIE DISPLAY AND CONTROL enum { SOURCE_AVI, SOURCE_CAM, SOURCE_UNKNOWN } g_video_source = SOURCE_UNKNOWN; int g_local_frame_slider = 20, g_local_frame_number = 0; //Move +/- 20 frames back and forth int suspend = 0; //Suspend frame processing when you click on local moves int g_global_frame_slider = 0; //Move around whole file CvCapture* capture = 0; IplImage *image=0, *imageA = 0,*imageB = 0,*frame = 0; IplImage *imageDiff = 0, *imageMarked = 0,*imageText1 = 0 ,*imageText2=0, *Ihsv=0; CvFont font; int skip_count = 0; //Skip Frames count int frame_number = 0; int frame_slider_jumps = 0; //Each tick on the global slider is worth this much int total_frames = 0; int go = 1; //Go mode: 0 single step, 1 continuous run int step = 0; //step forward a frame int display = 0; //If set to 1, update image displays //MULTI-WINDOW CvxMultiWin mw; //MOUSE CvPoint pt; int panelCol, panelRow, image_x, image_y, seg_image_x = 0, seg_image_y = 0; int left_down = 0, left_down_seg = 0, l_image_x, l_image_y, l_offset; CvPoint pt_low,pt_high; int drawing_rect_right = 0; int drawing_rect_left = 0; int rdown = 0; int ldown = 0; //OBJECTS char object_names[MAX_OBJECTS][64]; int number_of_objects = 0,ii,jj; //Callback mouse handler void on_mouse( int event, int x, int y, int flags,void *foo = NULL ) { if( !image ) return; panelCol = panelRow = image_x = image_y = 0; //Find out global which panel, x,y within the mw panel and image x,y within the raw displayed window mw.panelXY(x, y, panelCol, panelRow, image_x, image_y); //DRAW A RIGHT BUTTON RECTANGLE { if( event == CV_EVENT_RBUTTONDOWN ) { if((image_y < 0)||(image_x < 0)) { printf("Hit pannel border, skip\n"); } else { if((panelCol == 0) && (panelRow == 0)) { printf("Right Panel(%d %d) and image (%d %d)\n",panelCol,panelRow,image_x,image_y); pt_low = cvPoint(image_x,image->height - image_y); go = 0; step = 0; drawing_rect_right = 1; display = 1; rdown = 1; } } } if(drawing_rect_right && rdown) { if((image_y < 0)||(image_x < 0)) { printf("Hit pannel border, skip\n"); } else pt_high = cvPoint(image_x,image->height - image_y); display = 1; } if(event == CV_EVENT_RBUTTONUP) { if(drawing_rect_right) { rdown = 0; drawing_rect_right = -1; } printf("Right up (%d %d) panel(%d %d) image(%d %d)\n", x,y,panelCol,panelRow,image_x,image_y); } } //DRAW A LEFT BUTTON RECTANGLE { if( event == CV_EVENT_LBUTTONDOWN ) { if((image_y < 0)||(image_x < 0)) { printf("Hit pannel border, skip\n"); } else { if((panelCol == 0) && (panelRow == 0)) { printf("Left Panel(%d %d) and image (%d %d)\n", panelCol,panelRow,image_x,image_y); pt_low = cvPoint(image_x,image->height - image_y); go = 0; step = 0; drawing_rect_left = 1; display = 1; ldown = 1; } } } if(drawing_rect_left && ldown) { if((image_y < 0)||(image_x < 0)) { printf("Hit pannel border, skip\n"); } else pt_high = cvPoint(image_x,image->height - image_y); display = 1; } if(event == CV_EVENT_LBUTTONUP) { if(drawing_rect_left) { ldown = 0; drawing_rect_left = -1; } printf("Left up (%d %d) panel(%d %d) image(%d %d)\n", x,y,panelCol,panelRow,image_x,image_y); } } } //Callback Trackbar handler void onSlideLocalFrames(int pos) { if (!suspend) { suspend = 1; g_local_frame_number = frame_number; go = 0; //turn off continuous run skip_count = 0; //Turn off skip frame amounts } int new_position = g_local_frame_number + (pos - 20); if(new_position < 0) new_position = 0; if(new_position >= total_frames) new_position = total_frames - 2; cvSetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES, new_position ); frame_number = (int) cvGetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES ); step = 1; printf("Slider at %d position. Delta frames is %d, Frame to set to is %d\n", pos,pos-20,new_position); } void onSlideGlobalFrames(int pos) { int jumpto = pos*(total_frames/FRAME_TIME_RESOLUTION); if(pos >= FRAME_TIME_RESOLUTION) jumpto -= 2; printf("Slider at %d position. Go to frame %d\n", pos,jumpto); cvSetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES, jumpto ); step = 1; } //printt help to stdout void help() { printf( "ReadMovie //read movie from disk;\nReadMovie //Will read from camera\n" "Hot keys: \n" "====== Out/Help ======\n" "\tESC - quit the program\n" "\th - print this help menue\n" "====== Flow Control ======\n" "\tg - toggle \"go mode\"\n" "\ts - skip ahead 10%% of the frames\n" "\tS - skip back 10%% of the frames\n" "\tn - next frame\n" "\tp - previous frame\n" "====== Drawing ======\n" "\tRight mouse upper left panel draws red rectangle\n\t\tthen it enters a label. \"go\" to restart.\n" "\tLeft mouse upper left panel draws blue box\n" ); } // Calls: ReadMovie filename //Run movie file // ReadMovie //Use camera int main( int argc, char** argv ) { int c; //TEXT STUFF FOR WRITING TO MULTIWINDOW FRAMES. Watch out for font size (see cvInitFont below). char text_buf11[128],text_buf12[128],text_buf13[128],text_buf14[128],text_buf15[128]; text_buf11[0] = 0,text_buf12[0] = 0,text_buf13[0] = 0,text_buf14[0] = 0,text_buf15[0] = 0; char text_buf21[128],text_buf22[128],text_buf23[128],text_buf24[128],text_buf25[128]; text_buf21[0] = 0,text_buf22[0] = 0,text_buf23[0] = 0,text_buf24[0] = 0,text_buf25[0] = 0; CvPoint line1,line2,line3,line4,line5; line1.x=5, line2.x=5, line3.x=5, line4.x=5, line5.x=5; //From Camera if( argc == 1 || (argc == 2 && strlen(argv[1]) == 1 && isdigit(argv[1][0]))) { capture = cvCaptureFromCAM( argc == 2 ? argv[1][0] - '0' : 0 ); g_video_source = SOURCE_CAM; } else if( argc == 2 ) //Or from AVI { capture = cvCaptureFromAVI( argv[1] ); g_video_source = SOURCE_AVI; } if( !capture ) { fprintf(stderr,"Could not initialize capturing...\n"); help(); return -1; } help(); //PRIME THE IMAGE AQUISITION frame = cvQueryFrame(capture); if( !frame ) return -1; /* allocate all the buffers */ imageA = cvCreateImage( cvGetSize(frame), 8, 1 ); imageA->origin = 0; // Image at time "T" image = cvCreateImage( cvGetSize(frame), 8, 3 ); image->origin = 0; // Image at time "T" Ihsv = cvCloneImage(image); //HSV image imageB = cvCloneImage(imageA); // Image at time T-1 imageDiff = cvCloneImage(imageA); // Image at time T-1 imageMarked = cvCloneImage(imageA); // Image at time T-1 imageText1 = cvCloneImage(imageA); // Image at time T-1 cvSetZero(imageText1); imageText2 = cvCloneImage(imageA); // Image at time T-1 cvSetZero(imageText2); imageText1->origin = 1; //So that font comes out right side up imageText2->origin = 1; int myWidth = imageA->width; int myHeight = imageA->height; int nChan = imageA->nChannels; printf("Width=%d, Height=%d, Channels=%d\n",myWidth,myHeight,nChan); cvCvtColor(frame,imageB,CV_BGR2GRAY); frame = cvQueryFrame(capture); cvCopy(frame,image); cvCvtColor(frame,imageA,CV_BGR2GRAY); frame_number = 2; //SET UP THE MULTI-WINDOW mw.initialize("Movie",myWidth,myHeight, 3,3,8,3,0,IPL_ORIGIN_BL); mw.set_label( 0,0, "Raw"); mw.set_label( 1,0, "t"); mw.set_label( 2,0, "t-1"); mw.set_label( 0,1, "Diff"); mw.set_label( 1,1, "Marked"); mw.set_label( 2,1, "HSV"); mw.set_label( 0,2, "Text1"); mw.set_label( 1,2, "Text2"); //Create Mouse Handlerfor multi-win cvSetMouseCallback( "Movie", on_mouse ); //Create Trackbars for multi-win if( g_video_source == SOURCE_AVI ) { total_frames = (int) cvGetCaptureProperty( capture, CV_CAP_PROP_FRAME_COUNT ); frame_slider_jumps = total_frames/FRAME_TIME_RESOLUTION; printf("TOTAL FRAMES = %d\n", total_frames ); printf("FRAME NUMBER = %d\n", frame_number ); cvCreateTrackbar( "Local_jmp", "Movie", &g_local_frame_slider, 40, onSlideLocalFrames ); cvSetTrackbarPos("Local_jmp", "Movie", g_local_frame_slider ); cvCreateTrackbar( "Global_jmp", "Movie", &g_global_frame_slider, FRAME_TIME_RESOLUTION, onSlideGlobalFrames ); } frame_number = (int) cvGetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES ); printf("FRAME = %d\n", frame_number ); //Create Window for color display cvNamedWindow( "ReadMovie", 0 ); //SET UP IMAGE TEXT cvInitFont(&font, CV_FONT_HERSHEY_PLAIN, 1.0, 1.0); line1.y = myHeight - myHeight/6; line2.y = myHeight - 2*myHeight/6; line3.y = myHeight - 3*myHeight/6; line4.y = myHeight - 4* myHeight/6; line5.y = myHeight - myHeight/6; //CAPTURE LOOP for(;;) { if(go || step) { //IF SKIPPING FORWARD while(skip_count != 0) { frame = cvQueryFrame(capture); skip_count -= 1; } //GET NEXT FRAME frame = cvQueryFrame( capture ); if( !frame ) break; frame_number = (int) cvGetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES ); g_global_frame_slider = frame_number/frame_slider_jumps; if(!(frame_number % frame_slider_jumps)) { cvSetTrackbarPos("Global_jmp", "Movie", g_global_frame_slider ); } // printf("%d\n",g_global_frame_slider); cvCopy(imageA,imageB,0); cvCvtColor( frame, imageA, CV_BGR2GRAY ); cvCopy(frame,image); cvCvtColor( image, Ihsv, CV_BGR2HSV ); cvAbsDiff(imageA,imageB,imageDiff); display = 1; step = 0; } //END if going or stepping if(display) { display = 0; //LEFT RECTANGLE { if(drawing_rect_left) { cvCopy(frame,image); cvRectangle( image, pt_low, pt_high,CV_RGB(0,255,0),2); } if(drawing_rect_left < 0) { drawing_rect_left = 0; colorRectangle(Ihsv, pt_low, pt_high,1,255.0); colorRectangle(imageDiff, pt_low, pt_high,0,255.0); // go = 1; //Automatically restart } } //RIGHT RECTANGLE if(drawing_rect_right) { cvCopy(frame,image); cvRectangle( image, pt_low, pt_high,CV_RGB(255,0,0),2); } if(drawing_rect_right < 0) { drawing_rect_right = 0; //CORRECT THE POINTS int tmppt; if(pt_low.x > pt_high.x) { tmppt = pt_low.x; pt_low.x = pt_high.x; pt_high.x = tmppt; } if(pt_low.y > pt_high.y) { tmppt = pt_low.y; pt_low.y = pt_high.y; pt_high.y = tmppt; } printf("\nEnter Object label[%d] (64 chars max). RETURN to enter; ESC to cancel; BACKSPACE works\n",number_of_objects); ii = 0; c = 0; while((c != 27)&&(c != 13)&&(ii < 64)) { c = cvWaitKey(); if(c == 13) break; //cr if(c == ' ') continue; //space (ignored) if(c == '\t') continue; //tab if(c == 8) //back space { printf("\r"); for(jj = 0; jj<=ii; jj++) printf(" "); printf("\r"); ii -= 1; if (ii <= 0) ii = 0; object_names[number_of_objects][ii] = 0; } else { object_names[number_of_objects][ii] = c; ii++; object_names[number_of_objects][ii] = 0; } printf("%s\r",object_names[number_of_objects]); } //END WHILE //IF DIDN'T ESCAPE OUT, PROCESS THE RECTANGLE REGION if(c != 27) //!ESC enter label and DO WHAT YOU WANT TO DO WITH THE RECTANGLE HERE { number_of_objects++; if(number_of_objects >= MAX_OBJECTS) number_of_objects = MAX_OBJECTS - 1; // go = 1; Automatically restart } } //UPDATE TEXT cvPutText(imageText1, text_buf11, line1, &font, cvScalar(0.0 )); //clear it sprintf(text_buf11,"Frame# = %d ",frame_number); cvPutText(imageText1, text_buf11, line1, &font, cvScalar( 255.0 ));//write new //PAINT IMAGES UP // CV_SWAP( prev_grey, grey, swap_temp ); cvShowImage( "ReadMovie", frame ); mw.paint(image,0,0,false); mw.paint(imageA,1,0,false); mw.paint(imageB,2,0,false); mw.paint(imageDiff,0,1,false); mw.paint(imageMarked,1,1,false); mw.paint(Ihsv,2,1,false); mw.paint(imageText1,0,2,false); mw.paint(imageText2,1,2,true); //paint and output } //Get keyboard input int skipto; c = cvWaitKey(10); if( c == 27 ) break; if(c != -1) //some key was pressed { suspend = 0; //out of local frame slider operation } switch( c ) { case 'h': //Help help(); break; case 's': //Skip forward 10% of the frames skip_count = total_frames/10; printf("Skiping froward %d frames\n",skip_count); step = 1; break; case 'S': //Skip back 10% fo the frames skip_count = 0; skipto = frame_number - (total_frames/10); if(skipto < 0) skipto = 0; frame_number = skipto; cvSetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES, (double)skipto ); printf("Skipping forward %d frames to %d\n",total_frames/10,skipto); step = 1; break; case 'g': //Go mode if(go == 1) { go = 0; printf("Go off!\n"); } else { go = 1; printf("Continuous Go mode\n"); } break; case 'n': //Next frame single step step = 1; go = 0; printf("Single step forward\n"); break; case 'p': //Skip back a frame skip_count = 0; skipto = frame_number - 2; if(skipto < 0) skipto = 0; frame_number = skipto; cvSetCaptureProperty( capture, CV_CAP_PROP_POS_FRAMES, (double)skipto ); printf("Single step backwards\n"); step = 1; break; default: ; } } cvReleaseCapture( &capture ); cvDestroyWindow("ReadMovie"); return 0; } //This function is just to show how to access a rectangle in an image using pointers // point uper left must be smaller in both coordinates (origin top left) than the point in lower left // Supports only 8U and and 32F // putVal = the number to put in // -1 bad int colorRectangle(IplImage *I, CvPoint pt_upperL, CvPoint pt_lowerR, int channel, float fputVal) { int nChannels = I->nChannels; if(channel >= nChannels) return -1; int x1,y1,x2,y2; if(pt_upperL.x > pt_lowerR.x) { x1 = pt_lowerR.x; x2 = pt_upperL.x; } else { x1 = pt_upperL.x; x2 = pt_lowerR.x; } if(pt_upperL.y > pt_lowerR.y) { y1 = pt_lowerR.y; y2 = pt_upperL.y; } else { y1 = pt_upperL.y; y2 = pt_lowerR.y; } unsigned char *cptr,cputVal; float *fptr; int floatImage = 0; if(I->depth == IPL_DEPTH_32F) floatImage = 1; else { if((fputVal < 0.0)||(fputVal > 255.0)) cputVal = 255; else cputVal = (unsigned char)(fputVal + 0.5); } int width = I->width; int widthStep = I->widthStep; // int height = I->height; int x,y; for(y=y1; yimageData + y*width*nChannels + x1*nChannels + channel; else cptr = (unsigned char *)(I->imageData + y*widthStep + x1*nChannels + channel); for(x=x1; x