handled unk token while decoding

ancillary_functions_anuvaad/output_cleaner.py

@@ -16,6 +16,8 @@ def purnaviram_applier(src,tgt):
     For english to hindi translation
     '''
     try:
+        if tgt is None or len(tgt.split()) == 0:
+            return tgt
         if len(src.split()) < 5:
             return tgt
         if src.endswith('.') and tgt.endswith('।'):

tools/sp_enc_dec.py

@@ -97,7 +97,7 @@ def decode_line(load_model,line):
         line = line[0]+line[1:-1].replace(']',"")+line[-1]  
         logger.info("decoding using sp model {}".format(load_model))
         if "<unk>" in line:
-            line = line.replace("<unk>","'<unk>'")
+            line = line.replace("<unk>","")
         return sp.DecodePieces(eval(line))
     except Exception as e:
         logger.error("something went wrong! {}".format(e))