From c46367b2226a7eb188abd454b2e6d6b92bda9f79 Mon Sep 17 00:00:00 2001 From: Burhan Two Date: Wed, 4 Jul 2018 19:08:52 +0500 Subject: [PATCH] Late Fusion Recognition --- post_processing.c | Bin 0 -> 8528 bytes processing.h | 12 ++ run.c | 442 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 454 insertions(+) create mode 100755 post_processing.c create mode 100644 processing.h create mode 100644 run.c diff --git a/post_processing.c b/post_processing.c new file mode 100755 index 0000000000000000000000000000000000000000..29b78d65777103e696013e6adfa2e726f3345e53 GIT binary patch literal 8528 zcmeHMYit}>6~4RcSL$@Vc9Pb4h^!zHPN}Es)G-ce$?W+uJ*e-&*4$csxi_cf~`A_((_lLy2f> zBA&`kwN7<*w05*B*|f4-_8a}9ZQsE`Srd<=7c|*Fkhli0{GZO3E`O_U;`VgY3%}m* z_*-B7{*8b2Vj1>_=k1vVuR~O+yAZ=Mu7@YkbLsle-ul|fV^4hj22^MGT$ z!?X{Xk&t-3@LBM00FHjF{xj0PK~&K?Vm|`r&xmESu}oytz-fC}=m#eCf!KIFYs4~L zi3pTsg&qv|>(N+-;BbFeBAtqbBO{5J(Dm_TI;CfgNXF20)70KDo{Aen9|M8aj$(Bk zK*v1MlaY8z=&@+Th(N(eHf!PlrJ|xQ6by9fyOrI_9%{b17#(tT;M@G0l4D?+G$QM8^O7YN42>> zYqM|PJRA;QSP(FC+J*D;H7G9NdRYD`5dS{bu)SM|v-_c;gIX)jyEePx*5sU0gLb6};86 zw}$OvXwj^H=Fwa0yh7v}2Z6Ypf`$#fR{ckS9Pi1R&UxU2sm<;ex$0{#0I%Te783Gr z3|&0|oMLVT7>__*@3RF$4l` z&-2cJyZJd+kK22?D&TH;wpw%hPSt4c&e__2x0-Qx`rSUiyCvZEf*sfg+;y_u&mtfC zVeTeW?xwpVaCZdmj=muEk zwo-W=$t|o?Ip6cR{2#0NG@6`oswfI~z7?0L<2eq~Wv7Zl+#}y`9wJ#!vYF&2k}L{m zIynAd;@?2|+Y37XPUE={)BUD^hNv>f_;Qu0bGx}dIDE>)I{aMA{=821@Uts*9^ad! zze43i|DOxK$66~Iqkc}3w3hdEb$!CyGB}b;89DDBrCss0cI2ei_NBHC#n-O1`F5EF z-rc^o4qu0FuhoZDcfo4wbgEVu?upLAy}7i|1to^9eN|!Il-pMq)=N2FBY0hv!Unh7yl;f~EZN||Bw^-{v4tXxvU59P$?-et)xVsRqDX-V&-Y5NggvWXx zlz3wy-catRNvO8=8;Wt15`qhMPaW;+5*YfmN#J$Bnq1;Oh_Be;_Y1zam2zAHb`|a0 za(uIBvBfLz#mI9B{wfu{;C`$UxAH~A%;ERWVbgy6&N(4?vS67P^o+Q$d{-61t>0Tb z(>qG;@BCdjBJs-cJVAbjE&fkQ`^tG)t6$%h_D#Z5r$VlnehgW={-TB5_?(l}%JuUb z>8Emj-<0+nD%Qz$!VB>^slzoz`l+1vP0%sBxQY(A7ve3}^{@@_Mqu#zuOj~+1Kcj& z;zN7qX5q7}tCyvH<^CF#cyT}S-ET(XmHYis@{iZ=(i8;TTMAHq2ypDLwO@aQa=SRv zF==19{wD!%g7t}E1x)fT3Gx)+UIZ-Z+hmW?2u%EY`KrVvRQ9^KhM%Q1a50)OvPN!f zOc{j$(1ET2Jrq0=hOkmG_@o<2eH6n_5SG%TX?;AA9*HFMsFBWO^+;|?jHZ*v6EPze zRX)CFZ@aybj6&&=OeQi7u_YrjEygmDWK56dlF4bHlxRAJt;$Swy?4Oh-=p^&?8bnX zY$oR+kg4}*WT15q2)+C9gZ}p!=mw&p3BCf!i<&a(QI0uh@_%25(j}fZk1gWPshL-htx{LYi0~?G@Fxc{D5$u zjMt&9jO10s?^LxkP(Y@1n1%@h(#V3N-Ax{6n`MtYDC6K8&Bnw zkz5=C(DA5HP@9NkCxjB6PJusD8X2=>GM354)2T8=hq6p85kUgwjwcME$XKNUS!Fy8 zsu7!lx9qTzNz3@85}Tl9G7$wQteI4Ey_gKn9~%WTs3ww(k3t*L2D%3XgPjV*I)wr& zHwmk;YzO^!3jXfE7Ax#eK2P!gH~5^*-qE%+eFT(RczAz@ohoR>|6_QiR($dIlaV&a zl=As}W{|@jY)I0|?*{xfoQ1jmGfq`no+027sTJ=7eC{S#9E1#>{h5E+se*?01Sy1< zI$Q%PW|i<0PGnNVG zdfDH1EPTEXSK+3B#P>o|tL*;|pa84Dvj`?Wr+A1@(`XjLn{}2y0u1>)|NMVL{(s_w z1*Vx{9>aK!E9LXKU#0qg$A%=W +#include +#include +#include + +#include + +#include +#include + +#include "processing.h" + +#define BLACK 20.0 +#define YELLOW 70.0 + +#define min_f(a, b, c) (fminf(a, fminf(b, c))) +#define max_f(a, b, c) (fmaxf(a, fmaxf(b, c))) + + +int xflag = 0; +int last_index =0; +int skip_frames = 12; +char *temptext = ""; +int xcounter = 0; +int searchword(char *text, char *word) { + int i; + + while (*text != '\0') { + while (isspace((unsigned char) *text)) + text++; + for (i = 0; *text == word[i] && *text != '\0'; text++, i++); + if ((isspace((unsigned char) *text) || *text == '\0') && word[i] == '\0') + return 1; + while (!isspace((unsigned char) *text) && *text != '\0') + text++; + } + + return 0; +} + +void late_fusion(PIX* img, int index){ + + char *text; + char *writetext; + + char write_path[500]; + text = russian_ocr(img,index); + writetext = russian_ocr(img,index); + printf("%s", text); + if(xcounter == 0){ + temptext = text; + xcounter++; + } + + //printf("%s", temptext); + int totalmatch = 0; + char *word = strtok(text, " "); + //printf("%s", word); + + while (word != NULL) { + if (searchword(word, temptext)) { + printf("Match: %s\n", word); + sprintf(write_path,"im%04d.jpg",index); + pixWrite(write_path,img,IFF_JFIF_JPEG); + writetofile(writetext, index); + } + else{ + temptext = text; + } + word = strtok(NULL, " "); +} + + + + + +} +void rgb2lab(float R, float G, float B,float *L, float *a, float *b){ + //Conversion to the CIE-LAB color space to get the Luminance + float X, Y, Z, fX, fY, fZ; + + X = 0.412453*R + 0.357580*G + 0.180423*B; + Y = 0.212671*R + 0.715160*G + 0.072169*B; + Z = 0.019334*R + 0.119193*G + 0.950227*B; + + X /= (255 * 0.950456); + Y /= 255; + Z /= (255 * 1.088754); + + if (Y > 0.008856) + { + fY = pow(Y, 1.0/3.0); + *L = 116.0*fY - 16.0; + } + else + { + fY = 7.787*Y + 16.0/116.0; + *L = 903.3*Y; + } + + if (X > 0.008856) + fX = pow(X, 1.0/3.0); + else + fX = 7.787*X + 16.0/116.0; + + if (Z > 0.008856) + fZ = pow(Z, 1.0/3.0); + else + fZ = 7.787*Z + 16.0/116.0; + + *a = 500.0*(fX - fY); + *b = 200.0*(fY - fZ); + + if (*L < BLACK) { + *a *= exp((*L - BLACK) / (BLACK / 4)); + *b *= exp((*L - BLACK) / (BLACK / 4)); + *L = BLACK; + } + if (*b > YELLOW) + *b = YELLOW; +} + +int detect_stopper_color(int r, int g, int b){ + +//For Detecting Blue Color + if (r>=20 && r<=35){ + if(g >=75 && g<=135){ + if(b>=130 && b<=180){ + return 1; + } +} +} + else return 0; +} + +void die(const char *errstr) { + fputs(errstr, stderr); + exit(1); +} + +void writetofile(char* text, int index){ + FILE * fp; + char write_path[100]; + //Writing to a File + sprintf(write_path,"txt%04d.txt",index); + fp = fopen(write_path, "w"); + fprintf(fp, "%s", text); + fclose(fp); + + fputs(text, stdout); + +} +char* russian_ocr(PIX* img, int index){ + TessBaseAPI *handle; + char *text; + + + + handle = TessBaseAPICreate(); + + if(TessBaseAPIInit3(handle, NULL, "rus") != 0) + die("Russian Language Not Set for Tesseract\n"); + + TessBaseAPISetImage2(handle, img); + + if(TessBaseAPIRecognize(handle, NULL) != 0) + die("Error in Tesseract recognition\n"); + + if((text = TessBaseAPIGetUTF8Text(handle)) == NULL) + die("Error getting text\n"); + + + + //Writing to a File + //sprintf(write_path,"txt%04d.txt",index); + //fp = fopen(write_path, "w"); + //fprintf(fp, "%s", text); + //fclose(fp); + + //Displaying it + //fputs(text, stdout); + return text; + TessDeleteText(text); + //TessBaseAPIEnd(handle); + //TessBaseAPIDelete(handle); + +} + +void _process_frame_tickertext(AVFrame *frame, int width, int height, int index){ + printf("index%d\n", index); + if (index >1600){ + PIX *im; + PIX *edge_im; + PIX *lum_im; + PIX *feat_im; + char *subtitle_text=NULL; + im = pixCreate(width,height,32); + lum_im = pixCreate(width,height,32); + feat_im = pixCreate(width,height,32); + int i,j; + + + + for(i=(92*height)/100;ilinesize[0]; + int r=frame->data[0][p]; + int g=frame->data[0][p+1]; + int b=frame->data[0][p+2]; + pixSetRGBPixel(im,j,i,r,g,b); + float L,A,B; + rgb2lab((float)r,(float)g,(float)b,&L,&A,&B); + if(L > 85) + pixSetRGBPixel(lum_im,j,i,255,255,255); + else + pixSetRGBPixel(lum_im,j,i,0,0,0); + } + } + + + + //Handle the edge image + edge_im = pixCreate(width,height,8); + edge_im = pixConvertRGBToGray(im,0.0,0.0,0.0); + edge_im = pixSobelEdgeFilter(edge_im, L_VERTICAL_EDGES); + edge_im = pixDilateGray(edge_im, 21, 11); + edge_im = pixThresholdToBinary(edge_im,45); + + for(i=92*(height/100);i0) + pixSetRGBPixel(feat_im,j,i,255,255,255); + else + pixSetRGBPixel(feat_im,j,i,0,0,0); + } + } + + char write_path[100]; + int num_of_occr = 0; + int stop = 0; + char *text; + for(i=(92*height)/100;ilinesize[0]; + int r=frame->data[0][p]; + int g=frame->data[0][p+1]; + int b=frame->data[0][p+2]; + + //printf("r %d\n", r); + //printf("g %d\n", g); + // printf("b %d\n", b); + + + + //Detecting Blue Stoppers + //Skipping Thirty Frames and Detecting Blue Color + /* + if (detect_stopper_color(r,g,b) == 1 && index % 30 == 0){ + //printf("%s\n", "Stopper Detected"); + xflag = 1; + //Hardcoded Bouding Box Locations + BOX* box = boxCreate(100, 525, 615, 25); + PIX* pixd= pixClipRectangle(feat_im, box, NULL); + last_index = index; + writetofile(russian_ocr(pixd, index), index); + sprintf(write_path,"im%04d.jpg",index); + pixWrite(write_path,pixd,IFF_JFIF_JPEG); + boxDestroy(&box); + pixDestroy(&pixd); + stop = 1; + } + //Checking of text after blue stopper + else if(detect_stopper_color(r,g,b) == 0 && index %11 == 0 && xflag == 1){ + BOX* box = boxCreate(100, 525, 615, 25); + PIX* pixd= pixClipRectangle(feat_im, box, NULL); + writetofile(russian_ocr(pixd, index), index); + sprintf(write_path,"im%04d.jpg",index); + pixWrite(write_path,pixd,IFF_JFIF_JPEG); + boxDestroy(&box); + pixDestroy(&pixd); + xflag = 0; + stop = 1; + } +*/ + + BOX* box = boxCreate(100, 525, 615, 25); + PIX* pixd= pixClipRectangle(feat_im, box, NULL); + last_index = index; + late_fusion(pixd, index); + //writetofile(russian_ocr(pixd, index), index); + boxDestroy(&box); + pixDestroy(&pixd); + stop = 1; + } + +} + + pixDestroy(&lum_im); + pixDestroy(&im); + pixDestroy(&edge_im); + pixDestroy(&feat_im); + +} +} + +int main(int argc, char * argv[]) { + AVFormatContext *pFormatCtx = NULL; + int i, frame_count, videoStreamIdx; + AVCodecContext *pCodecCtx = NULL; + AVCodec *pCodec = NULL; + AVFrame *pFrame = NULL; + AVFrame *pFrameRGB = NULL; + AVPacket packet; + int frameFinished; + int numBytes; + uint8_t *buffer = NULL; + + AVDictionary *optionsDict = NULL; + struct SwsContext *sws_ctx = NULL; + + if(argc < 2) { + printf("Please provide a video file\n"); + return -1; + } + + // Register all formats and codecs + av_register_all(); + + // Open video file + if(avformat_open_input(&pFormatCtx, argv[1], NULL, NULL)!=0) + return -1; // Couldn't open file + + // Retrieve stream information + if(avformat_find_stream_info(pFormatCtx, NULL)<0) + return -1; // Couldn't find stream information + + // Dump information about file onto standard error + av_dump_format(pFormatCtx, 0, argv[1], 0); + + // Find the first video stream + videoStreamIdx = -1; + for(i=0; inb_streams; i++) { + if(pFormatCtx->streams[i]->codec->codec_type==AVMEDIA_TYPE_VIDEO) { + videoStreamIdx = i; + break; + } + } + + if(videoStreamIdx==-1) + return -1; // Didn't find a video stream + + // Get a pointer to the codec context for the video stream + pCodecCtx=pFormatCtx->streams[videoStreamIdx]->codec; + + // Find the decoder for the video stream + pCodec = avcodec_find_decoder(pCodecCtx->codec_id); + if(pCodec==NULL) { + fprintf(stderr, "Unsupported codec!\n"); + return -1; // Codec not found + } + + // Open codec + if(avcodec_open2(pCodecCtx, pCodec, &optionsDict)<0) + return -1; // Could not open codec + + // Allocate video frame + pFrame = av_frame_alloc(); + + // Allocate an AVFrame structure + pFrameRGB = av_frame_alloc(); + if(pFrameRGB==NULL) + return -1; + + // Determine required buffer size and allocate buffer + numBytes = avpicture_get_size(PIX_FMT_RGB24, pCodecCtx->width, pCodecCtx->height); + buffer = (uint8_t *)av_malloc(numBytes*sizeof(uint8_t)); + + sws_ctx = sws_getContext ( + pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, + pCodecCtx->width, pCodecCtx->height, PIX_FMT_RGB24, + SWS_BILINEAR, NULL, NULL, NULL + ); + + // Assign appropriate parts of buffer to image planes in pFrameRGB + // Note that pFrameRGB is an AVFrame, but AVFrame is a superset of AVPicture + avpicture_fill((AVPicture *)pFrameRGB, buffer, PIX_FMT_RGB24, pCodecCtx->width, pCodecCtx->height); + + // Read frames and count the number of frames + frame_count = 0; + while(av_read_frame(pFormatCtx, &packet)>=0) { + //Is this a packet from a video stream? + if(packet.stream_index==videoStreamIdx) { + // Decode the video frame + avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished, &packet); + + //Did we get a video frame? + if(frameFinished) { + // Increment frame count + frame_count++; + + // Convert the image from its native format to RGB + sws_scale( + sws_ctx, (uint8_t const * const *)pFrame->data, pFrame->linesize, 0, + pCodecCtx->height, pFrameRGB->data, pFrameRGB->linesize + ); + + _process_frame_tickertext(pFrameRGB,pCodecCtx->width,pCodecCtx->height,frame_count); +//_display_frame(pFrameRGB,pCodecCtx->width,pCodecCtx->height,frame_count); + + //Now you have the frame and can do whatever you wish to + + + } + } + + // Free the packet that was allocated by av_read_frame + av_free_packet(&packet); + } + + // Free the RBG image + av_free(buffer); + av_free(pFrameRGB); + + //Free the YUV frame + av_free(pFrame); + + //Close the codec + avcodec_close(pCodecCtx); + + //Close the video file + avformat_close_input(&pFormatCtx); + + return 0; +}