Upload 12 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Video005-Scene-043.mp4 filter=lfs diff=lfs merge=lfs -text

Final_ISRO_DenseNet201_Epoch50.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6b8145b4fe5a9d3ce9d54edfa35bcce6f27fb85a6581488fac068d2414bd5c9
+size 140107848

Train_Label.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2883f13fef0601ccf5e6624638dce5b1342685b41f7c94f34d5b2f30acb2eed
+size 198919

Video005-Scene-043.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55751881488b7d6c4f427f3394a881d3041ddd894d43fd4e544d01b784217a27
+size 1386587

densenet.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c721c4c5ff8eb454517432bb9ffc1397bb83c5d72f804de08c21c487cd8afb0a
+size 222191800

image_model_transfer.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc07714490afaa42f1b09170e0d27289d05bd6fb56666ca50676a26818da70a8
+size 75231432

main.py

@@ -0,0 +1,378 @@
+import streamlit as st
+import numpy as np
+import cv2
+import tensorflow as tf 
+from PIL import Image
+from keras.models import load_model
+from sklearn.preprocessing import LabelEncoder
+import pickle
+from  keras_preprocessing.sequence import pad_sequences
+from  keras.preprocessing.text import Tokenizer
+from sklearn.preprocessing import LabelEncoder
+from PIL import Image
+# from google.colab.patches import cv2_imshow
+
+def label_smoothing(y_true,y_pred):
+    
+     return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.1)
+def sparse_cross_entropy(y_true, y_pred):
+    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true,
+                                                          logits=y_pred)
+    loss_mean = tf.reduce_mean(loss)
+    return loss_mean
+model1 = load_model('densenet.h5',custom_objects={'label_smoothing': label_smoothing})
+image_model_transfer=load_model("image_model_transfer.h5")
+decoder_model=load_model("Final_ISRO_DenseNet201_Epoch50.h5",custom_objects={'sparse_cross_entropy': sparse_cross_entropy})
+
+class TokenizerWrap(Tokenizer):
+    """Wrap the Tokenizer-class from Keras with more functionality."""
+    
+    def _init_(self, texts, num_words=None):
+        """
+        :param texts: List of strings with the data-set.
+        :param num_words: Max number of words to use.
+        """
+
+        Tokenizer._init_(self, num_words=num_words)
+
+        # Create the vocabulary from the texts.
+        self.fit_on_texts(texts)
+
+        # Create inverse lookup from integer-tokens to words.
+        # word_index is a dictionary. its values are tokens and the keys are words
+        # opposite to index_to_word
+        self.index_to_word = dict(zip(self.word_index.values(),
+                                      self.word_index.keys()))
+
+    def token_to_word(self, token):
+        """Lookup a single word from an integer-token."""
+        word = " " if token == 0 else self.index_to_word[token]
+        return word 
+
+    def tokens_to_string(self, tokens):
+        """Convert a list of integer-tokens to a string."""
+        # Create a list of the individual words.
+        words = [self.index_to_word[token]
+                 for token in tokens
+                 if token != 0]
+        
+        # Concatenate the words to a single string
+        # with space between all the words.
+        text = " ".join(words)
+
+        return text
+    
+    def captions_to_tokens(self, captions_listlist):
+        """
+        Convert a list-of-list with text-captions to
+        a list-of-list of integer-tokens.
+        """
+        
+        # Note that text_to_sequences() takes a list of texts.
+        tokens = [self.texts_to_sequences(captions_list)
+                  for captions_list in captions_listlist]
+        
+        return tokens
+with open('Train_Label.pickle', 'rb') as efile:
+    labels=pickle.load(efile)
+with open('tokenizer.pkl', 'rb') as efile:
+    tokenizer=pickle.load(efile)
+
+le=LabelEncoder()
+labels=le.fit_transform(labels)
+
+def framing(video):#defining a small function named"framing" with a parameter "i" that's supposed to be provided for reading the video
+    fr = []#creating an empty list named fr
+    fr_pre=[]#creating an empty list named fr_pre
+    cap = cv2.VideoCapture(video)#reading the video file
+    while (cap.isOpened()):#This command builds a loop to check if the data is still being read from the video
+        ret,frame = cap.read()#reading the data tunnel,gives two output where one tells about presence of frames(here it's ret) & the other speaks frame data(here it's frame)
+        if ret == True:#checking for presence of frames
+            # cv2_imshow(frame)#displaying the frames
+            grayed = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)#Converting the frames to Grayscale from BGR
+            canned = cv2.Canny(grayed,320,320)#For extrating edges we use Canny Edge detection method
+            fr.append(frame)#Appending the read frame
+            fr_pre.append(canned)#Appending the edge extracted frames
+            # cv2_imshow(grayed)#Displaying the original frames
+            # cv2_imshow(canned)#Displaying the edge detected frames
+            k = cv2.waitKey(10) & 0XFF#this is an arrangement for displaying the video where the secs for which each frame needs to be displayed in given in the paranthesis
+            if k == ord('q'):#pressing 'q' key will close the video
+                break
+        else:
+            break
+    cap.release()#Here we release the resoures
+    cv2.destroyAllWindows()#Here we delete all the windows that were created during the program       
+    return fr_pre,fr 
+
+def difference_of_frames(frames):
+    diff = []#creatin a list variable
+    for i in range(0,len(frames)-1):#defining the range
+        diff.append(cv2.absdiff(frames[i],frames[i+1]))#appending the diff between frames to the list variable so we're supposed to get only the difference between frames
+    return diff
+
+def cal_threshold(diff):
+  mn = np.mean(diff)#This gives mean
+  st_d = np.std(diff)#This gives standard deviation
+  a = 4#Setting a random value we can modify it to any value 
+  ts = mn + (a * st_d)#defining the standard threshold value for the project/global threshold value
+  return ts
+
+def imp_frames(diff, ts, ogframes):
+  a_fr = []#Creating an empty list
+  for i in range(len(diff)):#Defining the for loop to be looped over all the frames obtained after finding the frames resulted from subtracting
+      mn = np.mean(diff[i])#Calculating the mean for each frame
+      st_d = np.std(diff[i])#Calculating the standard deviation for each frame
+      fr_ts = mn + (4*st_d)#Finding the threshold values for each frame/image
+      a_fr.append([i,fr_ts])#Appending the frame number & the threshold values
+  imp_fr = []#Creating an empty list
+  for i,ac_tr in(a_fr):#Defining the loop on the list obtained from above code
+      if ac_tr >= ts:#Comapring the threshold values to the standard threshold/global threshold values
+          imp_fr.append([i,ac_tr])#Appending the list with the imp frames based on their index & the values
+  key_fr = []#Creating an empty list
+  for i,_ in imp_fr:#Defining the loop over the list obtained from above code
+      key_fr.append(ogframes[i])#This extracts the frames based on the index of frames 
+  return key_fr
+
+def final_image(video):
+  frames,ogframes = framing(video)#calling function framing & then extracting the images 
+  diff=difference_of_frames(frames)  
+  ts=cal_threshold(diff) 
+  key_fr=imp_frames(diff, ts, ogframes)  
+  frame_no=key_fr[int(len(key_fr)/2)] #this is a frame
+  cv2.imwrite("Testing1.jpg",frame_no)
+  return "Testing1.jpg"
+  cv2.destroyAllWindows()
+
+def image_test(image_path):
+  image=Image.open(image_path)
+  image = image.resize((224,224)) 
+  image = np.array(image) 
+  image= np.expand_dims(image, axis=0)
+  return image
+
+def largest_indices(ary, n):
+    flat = ary.flatten()
+    indices = np.argpartition(flat, -n)[-n:]
+    indices = indices[np.argsort(-flat[indices])]
+    return indices
+
+mark_start = 'ssss'
+mark_end = ' eeee'
+
+token_start = tokenizer.word_index[mark_start.strip()]
+token_end = tokenizer.word_index[mark_end.strip()]
+
+def load_image(path, size=None):
+  """
+  Load the image from the given file-path and resize it
+  to the given size if not None.
+  """
+
+  # Load the image using PIL.
+  img = Image.open(path)
+
+  # Resize image if desired.
+  if not size is None:
+      img = img.resize(size=size, resample=Image.LANCZOS)
+      
+  img = np.array(img)
+  img = img / 255.0
+
+  # Convert 2-dim gray-scale array to 3-dim RGB array.
+  if (len(img.shape) == 2):
+      img = np.repeat(img[:, :, np.newaxis], 3, axis=2)
+  return img
+
+def greedy_search(image_path, max_tokens=30):
+    """
+    Generate a caption for the image in the given path.
+    The caption is limited to the given number of tokens (words).
+    """
+    # ---------------------------ENCODE IMAGE--------------------------------
+    # Load and resize the image.
+    image = load_image(image_path, size=(224,224))
+    
+    # Expand the 3-dim numpy array to 4-dim
+    # because the image-model expects a whole batch as input,
+    # so we give it a batch with just one image.
+    image_batch = np.expand_dims(image, axis=0)
+
+    # Process the image with the pre-trained image-model
+    # to get the transfer-values.
+    transfer_values = image_model_transfer.predict(image_batch)
+    
+    # -------------------------------------------------------------------
+    
+    
+    # Pre-allocate the 2-dim array used as input to the decoder.
+    # This holds just a single sequence of integer-tokens,
+    # but the decoder-model expects a batch of sequences.
+    shape = (1, max_tokens)
+    decoder_input_data = np.zeros(shape=shape, dtype=int)
+
+    # The first input-token is the special start-token for 'ssss '.
+    token_int = token_start #1
+
+    # Initialize an empty output-text.
+    output_text = ''
+
+    # Initialize the number of tokens we have processed.
+    count_tokens = 0
+
+    # While we haven't sampled the special end-token for ' eeee'
+    # and we haven't processed the max number of tokens.
+    while token_int != token_end and count_tokens < max_tokens:
+        # Update the input-sequence to the decoder
+        # with the last token that was sampled.
+        # In the first iteration this will set the
+        # first element to the start-token.
+        decoder_input_data[0, count_tokens] = token_int
+
+        # Wrap the input-data in a dict for clarity and safety,
+        # so we are sure we input the data in the right order.
+        x_data = \
+        {
+            'transfer_values_input': transfer_values,
+            'decoder_input': decoder_input_data
+        }
+
+        # Note that we input the entire sequence of tokens
+        # to the decoder. This wastes a lot of computation
+        # because we are only interested in the last input
+        # and output. We could modify the code to return
+        # the GRU-states when calling predict() and then
+        # feeding these GRU-states as well the next time
+        # we call predict(), but it would make the code
+        # much more complicated.
+        
+        # Input this data to the decoder and get the predicted output.
+        decoder_output = decoder_model.predict(x_data)
+#         print(decoder_output.shape) (1,30,15000) for every iteration
+        
+        # Get the last predicted token as a one-hot encoded array.
+        # Note that this is not limited by softmax, but we just
+        # need the index of the largest element so it doesn't matter.
+        token_onehot = decoder_output[0, count_tokens, :]
+#         print(token_onehot.shape) (15000, ) for every iteration
+        # Convert to an integer-token.
+        token_int = np.argmax(token_onehot)
+#         print(token_int) #the token of a word with the highest score
+        
+        # Lookup the word corresponding to this integer-token.
+        sampled_word = tokenizer.token_to_word(token_int)
+#         print(sampled_word)
+
+        # Append the word to the output-text.
+        output_text += " " + sampled_word
+
+        # Increment the token-counter.
+        count_tokens += 1
+
+    # This is the sequence of tokens output by the decoder.
+    output_tokens = decoder_input_data[0]
+#     print(output_tokens)
+    # Plot the image.
+    # plt.imshow(image)
+    # plt.show()
+
+    predicted_caption=output_text.split()
+    del (predicted_caption[-1])
+    output_text = " "
+    output_text = output_text.join(predicted_caption)
+
+    # Print the predicted caption.
+    # print("Predicted caption:")
+    # print(output_text)
+    # print()
+    return predicted_caption
+
+def beam_search(beam_index, image_path, max_tokens=30):
+    image = load_image(image_path, size=(224,224))
+    
+    # Expand the 3-dim numpy array to 4-dim
+    # because the image-model expects a whole batch as input,
+    # so we give it a batch with just one image.
+    image_batch = np.expand_dims(image, axis=0)
+
+    # Process the image with the pre-trained image-model
+    # to get the transfer-values.
+    transfer_values = image_model_transfer.predict(image_batch)
+    
+    token_int = [token_start]
+    start_word = [[token_int, 0.0]]
+    count_tokens = 0 
+    while len(start_word[0][0])<max_tokens:
+        temp = []
+        
+        for s in start_word:
+            par_caps = pad_sequences([s[0]], maxlen=max_tokens, padding='post')
+            preds = decoder_model.predict([transfer_values,par_caps], verbose=0)
+            token_onehot = preds[0, count_tokens, :]
+#             print(token_onehot.shape)
+            word_preds = np.argsort(token_onehot)[-beam_index:]
+#             print(word_preds.shape)
+            
+            for w in word_preds:
+                next_cap, prob = s[0][:], s[1]
+                next_cap.append(w)
+                prob += token_onehot[w]
+                temp.append([next_cap, prob])
+                  
+        start_word = temp
+        count_tokens+=1
+        # Sorting according to the probabilities
+        start_word = sorted(start_word, reverse=False, key=lambda l: l[1])
+        # Getting the top words
+        start_word = start_word[-beam_index:]
+                  
+    start_word = start_word[-1][0]
+    intermediate_caption = [tokenizer.token_to_word(i) for i in start_word]
+    final_caption = []
+    
+    for i in intermediate_caption:
+        if i != 'eeee':
+            final_caption.append(i)
+        else:
+            break
+
+    # final_caption = ' '.join(final_caption[1:])
+    return final_caption[1:]
+
+def generate_caption_any(image_path):
+  predicted_caption1=' '.join((greedy_search(image_path=image_path)))
+  predicted_caption2=' '.join(beam_search(beam_index=3,image_path=image_path))
+  predicted_caption3=' '.join(beam_search(beam_index=5,image_path=image_path))
+  return predicted_caption2
+  # show_image_using_path(image_path)
+
+
+
+def main():
+    st.title("ISRO Video Classification & Captioning")
+    st.write('In this project, we introduce a technique for video classification and captioning, harnessing a keyframe extraction method to streamline the process. Utilizing Densenet 201, our model is designed to classify videos by focusing on the most crucial frame, optimizing efficiency and performance. Users can experience our innovative approach by employing any of the provided three videos or by uploading additional ISRO footage to witness the improved model in action.')
+
+    video_options = {
+        "Video 1": "Video001-Scene-001.mp4",
+        "Video 2": "Video015-Scene-074.mp4",
+        "Video 3": "Video005-Scene-043.mp4",
+    }
+
+    selected_video = st.selectbox("Select a video to submit", list(video_options.keys()))
+    video_path = video_options[selected_video]
+
+    if st.button("Submit"):
+        st.video(video_path)
+        path=final_image(video_path)
+        image=image_test(path)
+        output_class=model1.predict(image)
+        caption=generate_caption_any(path)
+        indices=largest_indices(output_class, 3)
+        st.title('The predicted category is:')
+        st.write(le.inverse_transform(indices)[0])
+        st.title('Caption:')
+        caption = caption.capitalize()
+        st.write(caption)
+        # st.video(uploaded_file)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,8 @@
+streamlit
+tensorflow
+keras
+pandas
+numpy
+pillow
+opencv-python
+scikit-learn

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8dbc8b1feb6fa1526be183d3e5df1151c4e63e6bda4c1d24543d3bf56a13fe7
+size 38857