o SQ8jϳ @s ddlZddlmZmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlZddlmZddlZddlmZmZmZddlmZddlmZGdd d ejZd d Zed kr e Z!e!j"d de#dde!j"dddddde!j"dddddde!j"dddddde!j"dddddde!$Z%e&e'e%j(d)ej*Z+ej,-dej,.e+dddl/m/Z/ej,.e+d e+d <ej,0e+d se1e+d ej,0ej,2e+d e+d!se1ej,2e+d e+d!ej,2e+d e+d!d"Z3ej,2e+d e+d!d#Z4ej,2e+d e+d!d$Z5ej,2e+d e+d!d%Z6ej,2e+d e+d!Z7d&d'e+d(DZ8gZ9e:e+d)e:e+d*krfd+ge:e+d*e+d)<e;e+d*e+d)D]\Zd,d'e?e<De=qoe%j@dureAd-ej,0e3se1e3e:eBe3ZCeCd.e+d/kre/e+d0e+d/eCe+d1d2gd2ggd3e3dd4d'eDe+d/Dd5 ejEFn eGd6e+d/d7eAd8ej,0e4se1e4e:eBe4ZCeCd.e+d9kre/e+d0e+d9eCe+d1d:gd:ggd3e4dd;ejEFn eGdZHe+d0D]ZIeH>eeIe+d/e:e+d0d:d?d@qQe/eHe+d/eCe+d1dAd2gd2ggd3e5ddBd'eDe+d/Dd5 ejEFn eGdCe+d/d7eAdDej,0e6se1e6e:eBe6ZCeCd.e+d9kre+d>ZHe+d0D]ZIeH>eeIe+d9e:e+d0d:d?d@qe/eHe+d9eCe+d1dAd:gd:ggd3e6dd;ejEFn eGdEe+d9d7dFZJdGd'ee4KdHDZLgZMeDeJD]ZNe jOjP)eLe jQRde:eL\ZSZTeMUe:eTqeVeWe XeMdIdIdJe+dK<e+dKdLkrRdLe+dK<neYe+dKdLdMkradLe+dK<e%jdurej,0ej,2e7dNrye%jZdurdOd'ee3KdHDe+dPZ[ee[e+dKe+dQe9e8dRZ\dSd'ee4KdHDe+dPZ]ee]e+dKe+dQe9e8dRZ^dTd'ee5KdHDe+dPZ_ee_e+dKe+dQe9e8dRZ`dUd'ee6KdHDe+dPZaeeae+dKe+dQe9e8dRZbeAdVecZdeddurd+ZdneddWZdee\e:eBe3e+dKej,2e7dNejEerdXndYejEes$ednd+dZee`e:eBe5e+dKej,2e7d[ejEerAdXndYejEesJednd+dZee^e:eBe4e+dKej,2e7d\ejEergdXndYejEespednd+dZeebe:eBe6e+dKej,2e7d]ejEerdXndYejEesednd+dZneGd^e%jfdur ejgjdYd_Zhehie+dKd`Zjed+eje+dae+dbdcejdd`ddZkddfdgZldhdie+djmDZniZodkgepe+djmdlgD]Zqeqdkkrdmdneoeq<qdodneoeq<qej,2e7dNe+djdk<ej,2e7d[e+djdl<ee+dje+dpeneodqZrGdrdsdsejgjsjtZuecZdeddur9d+ZdneddWZdejgjsjveuerdeddedtZwe &e+duZxe ydvd'eDdexjzdejdd+DZxe {exjzd|e j}Z~ejgjsjvejgjseexee~e:e~dwZxe &ej,2e7d\Ze &ej,2e7d]Ze e ejzde {ejzdf|e j}Zejgjsjvejgjsee eefeee:edwZekjeweexe+dxe+dye+dzd{Zekjee+d!e+d d|eej,2e+d e+d!d}ej,2e+d e+d!d~dSdSdS)N)optimnn)tqdm)Path)generate_adversarial_texts augment_clipsmmap_batch_generator)compute_features_from_generator) AudioFeaturescseZdZ  d+fdd Zdd Zd,d d Z  d-ddZddZddZd.ddZ d/ddZ  d0dd Z d.d!d"Z d.d#d$Z d%d&Zddddgd'gd(dfd)d*ZZS)1Model`dnnNc srt||_||_||_ttjrdnd|_g|_ g|_ d|_ d|_ d|_ d|_|dkrOGdddtjGfdd d tj}|||||d |_n|d krbGd d d tj}||||_|d krzdd|_tjdd|_tjdd|_nddd}ddd} ddd} ||_| |_| |_d|_d|_tt|_|d krtjjj ntjj!|_"t#j$|j%dd|_&dS)Nzcuda:0cpurrcs$eZdZfddZddZZS)z Model.__init__..FCNBlockcs2tt|||_t|_t||_dSN) super__init__rLinear fcn_layerReLUrelu LayerNorm layer_norm)self layer_dim __class__Y/home/nk/hobo-godmode/plappi-mvp/.venv/lib/python3.10/site-packages/openwakeword/train.pyr9s  z)Model.__init__..FCNBlock.__init__cSs||||Sr)rrrrxr!r!r"forward?sz(Model.__init__..FCNBlock.forward__name__ __module__ __qualname__rr% __classcell__r!r!rr"FCNBlock8s r+cs(eZdZdfdd ZddZZS)Model.__init__..Netr cstt|_t|d|d|_t|_t |_ t fddt |D|_ t||_|dkrDt|_dSt|_dS)Nrr csg|]}qSr!r!.0i)r+rr!r" Iz8Model.__init__..Net.__init__..)rrrFlattenflattenrlayer1rrelu1r layernorm1 ModuleListrangeblocks last_layerSigmoidlast_act)r input_shapern_blocks n_classes)r+r )rr"rCs     $$Model.__init__..Net.__init__c SsD|||||}|jD]}||}q|||}|Sr)r5r6r4r3r9r<r:)rr$blockr!r!r"r%Ms   #Model.__init__..Net.forward)r r r&r!r+rr"NetBs rD)r>r?rnncs&eZdZdfdd ZddZZS)r,r csXttj|ddddddd|_td||_|dkr%t|_ dSt|_ dS) N@T) num_layers bidirectional batch_firstdropoutrr ) rrrLSTMr4rlayer2r;rlayer3)rr=r?rr!r"rVs $r@cSs*||\}}|||dddfS)NrF)r4rPrO)rr$outhr!r!r"r%]srBr r&r!r!rr"rDUsr cSs||dkS)Ng)sum)predyr!r!r"dz Model.__init__..binary)task?cSsNtjjj|dd}|dk}||jddd||jddd|k@k}|SNr dimraxistorchr functionalsoftmaxargmaxmaxrT)prV thresholdprobsneg_ndcsfpr!r!r" multiclass_fphs0z%Model.__init__..multiclass_fpcSsVtjjj|dd}|dk}||jddd||jddd|k@k|}|Sr\ra)rgrVnegative_class_labelrhripos_ndcsrcllr!r!r"positive_class_recallns z-Model.__init__..positive_class_recallcSsJtjjj|dd}|jdd|k}||jdd||k|}|S)Nr r]r_)rbrrcrdrerT)rgrVrmri pos_predsaccr!r!r"positive_class_accuracyus$z/Model.__init__..positive_class_accuracy-C6?)lr)r[)rr[r)'rrr?r=seconds_per_examplerbdevicecuda is_available best_modelsbest_model_scores best_val_fpbest_val_accuracybest_val_recallbest_train_recallrModulemodelrk torchmetricsRecallrecallAccuracyaccuracyrrn_fpval_fp collections defaultdictlisthistoryrcbinary_cross_entropy cross_entropylossrAdam parameters optimizer) rr?r= model_typerr>rwrDrlrprsrrCr"rsB         zModel.__init__cCs |jdkrt|j|dSdS)z> Saves the weights of a trained Pytorch model r N)r?rbsaver)r output_pathr!r!r" save_models zModel.save_modelcs||jdkrtjj|jdt|jd||gddS|jdkrBGfdddtj }tjj|t|jd||gddSdS)Nr rr) output_namescs&eZdZfddZddZZS)zModel.export_to_onnx..Mcstjd|_dS)Nr)rrrtor)r objr!r"rs z(Model.export_to_onnx..M.__init__cSstjjj||ddS)Nr r])rbrrcrdrr#r!r!r"r%z'Model.export_to_onnx..M.forwardr&r!rrr"Msr) r?rbonnxexportrrrandr=rr)rr class_mappingrr!rr"export_to_onnxs     zModel.export_to_onnxrrIMbP?c Cstd|dttj|||t|||}|||}|dkr/t|||k||}t||k||}|S)Nr[r r)npcospifloatwhere) r global_step warmup_stepshold total_stepsstart_lr target_lr learning_rate warmup_lrr!r!r"lr_warmup_cosine_decays  zModel.lr_warmup_cosine_decaycCs ||Srrr#r!r!r"r% z Model.forwardcCstj|jd|jddS)NrSr) input_sizerx) torchinfosummaryrr=rr!r!r"rrz Model.summarycCs|dur|j}t|d}|}|D] }||d9<q|D]}|}|D] \}}|||7<q+q!|D] }||t|<q;|||S)zHAverages the weights of the provided models together to make a new modelNr)r{copydeepcopy state_dictitemslenload_state_dict)rmodelsaveraged_modelaveraged_model_dictkeyr model_dictvaluer!r!r"average_modelss zModel.average_models皙&@r[皙?cs~dgtj}|D]Q}|dj|dj}}ttjtjddD]0\} } t | |} || | |d || <Wdn1sUwYq*q fdd|D}fd dt|D} fd d| D} t | |krt d |d dSj| t| }j| t| d }t d|dt| dt | d|| t| |S)a? Select the top model based on the false positive rate on the validation data Args: false_positive_validate_data (torch.DataLoader): A dataloader with validation data n (int): The number of models to select Returns: list: A list of the top n models rr z,Find best checkpoints by false positive ratetotaldesc.NNcsg|]}|qSr!r!)r.rk) val_set_hrsr!r"r0r1z,Model._select_best_model..csg|] \}}|kr|qSr!r!)r.ndxrk)max_fp_per_hourr!r"r0scsg|] }j|dqS) val_recall)r|)r.rrr!r"r0zNo models with recall >= z found!training_step_ndxzBest model from training step z out of zmodels has recall of z and false positive rate of )rr{rrxr enumeraterbno_gradrkdetachrnumpyrfloggingwarningrrer|info)rfalse_positive_validate_datarr min_recallfalse_positive_ratesbatchx_valy_valmdl_ndxrval_pscandidate_model_ndxcandidate_model_recall best_modelbest_model_training_stepr!)rrrr"_select_best_models4 " * zModel._select_best_modelPrc CsNd}tdd}td|t|} t|t|d|dtj} |j||||| | |d|d||d td |d }|d }|j |krS|d }td td|t|} td|dtj } |j||||| | |d|d||d td|d }|j |kr|d }td td|t|} td|dtj } |j||||| | |d|d||d tdt |j dd} t |j dd} t |j dd } g}t |j|jD]\}}|d| kr|d| kr|d| kr||qt|dkr|j|d}n|j}tt|D]}|d|j|d|j}}||}q|||d}|||dtj}d}|D]"}|d|j|d|j}}||}||||d7}qY||}Wdn 1swYtd|d|d|d|S)zA sequence of training steps that produce relatively strong models automatically, based on validation data and performance targets provided. After training merges the best checkpoints and returns a single model. rz################################################## Starting training sequence 1... ##################################################rtr g?) XX_valfalse_positive_val_data max_stepsnegative_weight_schedule val_stepsr hold_stepsrurz################################################## Starting training sequence 2... ################################################## rHzCIncreasing weight on negative examples to reduce false positives...z################################################## Starting training sequence 3... ##################################################zBMerging checkpoints above the 90th percentile into single model... val_accuracyZr val_fp_per_hrr)rrNz( ################ Final Model Accuracy: z Final Model Recall: z' Final Model False Positives per Hour: z ################ )rrrlinspaceinttolistastypeint64 train_modelr}int16 percentilerzipr{r|appendrrrrbrrrxrrrrrrk)rX_trainrrstepsmax_negative_weighttarget_fp_per_hourrruweightsraccuracy_percentilerecall_percentile fp_percentilerrscorecombined_modelrr$rVrcombined_model_recallcombined_model_accuracycombined_model_fprrcombined_model_fp_per_hrr!r!r" auto_trains "            " $"  zModel.auto_trainc Cst|dkr |d}||j}g}t|ddD]H}|d}g}td|jdddD]}||dd||dddfq+t|}|durO| |}n||}|| dqt |S) a Predict on Tensors of openWakeWord features corresponding to single audio clips Args: features (torch.Tensor): A Tensor of openWakeWord features with shape (batch, features) model (torch.nn.Module): A Pytorch model to use for prediction (default None, which will use self.model) Returns: torch.Tensor: An array of predictions of shape (batch, prediction), where 0 is negative and 1 is positive rrzPredicting on clips)rrr rN)rrrxrr8shaperrbvstackrrrrr)rfeaturesr predictionsr$rr/predsr!r!r"predict_on_featuresps  &   zModel.predict_on_featurescCs2tddd}|j|dd}|jt||d}|S)a Predict on Tensors of 16-bit 16 khz audio data Args: clips (np.ndarray): A Numpy array of audio clips with shape (batch, samples) model (torch.nn.Module): A Pytorch model to use for prediction (default None, which will use self.model) Returns: np.ndarray: An array of predictions of shape (batch, prediction), where 0 is negative and 1 is positive r)rxncpur batch_sizer)r embed_clipsrrb from_numpy)rclipsrFrrr!r!r"predict_on_clipss zModel.predict_on_clipscCst|jdkr tdtdtj||ddt|}t j j | dt |jdtj||ddd d S) zDSaves the trained openwakeword model to both onnx and tflite formatsr zExporting models to both onnx and tflite with more than one class is currently not supported! Use the `export_to_onnx` function instead.z#### Saving ONNX mode as '.onnx'rr ) opset_versionN)r? ValueErrorrrospathjoinrrrbrrrrr=)rr model_name output_dir model_to_saver!r!r" export_models   zModel.export_modelrtc * Cs||j|j|jd} d} tg|j}tg|j}tt|d|ddD]7\}}|d|j|d|j}}|dtj}|jj D]}|j ||||| d|d<qO|j ||}||dk| dk@}||dk| d k@}t ||dk| dk@||dk| d k@f}|dtj}t ||f}t|dkrt|jd|d}|dk}d||<|d}n|jdkrt|jd||}|dk}d||<|d}|jddkr|||jdkr|n|||j}|| }| |jd7} |jdd kr|}|}| d kr/| d7} t ||f}t ||f}nT||jd} d} |jd ||||jdkrU|n|}|j|7_|jd |||tg|j}tg|j}|| vr|dkr|durd}t|D]:\}}t(|d|j|d|j}} ||}!|||!| d7}Wdn 1swYq|| }"|jd |"|| vrh|dkrh|durhd}#d}$t|D]_\}}tM|d|j}g}%td|jdddD]}&|%|dd|&|&dddfqt|%}%||%}'t |'dkrF|#d7}#n|$d7}$Wdn 1sUwYq|jd|#|#|$|| vr|dkr|durt|D]R\}}t@|d|j|d|j}} ||}!||!| d}(|!|!| dtj"})||!| d}Wdn 1swYq{|jd|)|jd|(|jd||| vr[|dkr[|jddt#$|jddkr[|jddt#$|jddkr[|j%t&'|j|j(||jdd|jdd|jdd|j)d dgdd|jdd|_*|jdd|_+||dkredSq-dS)Nr rTrainingrr)rrrrrurg+?rrrrrr[positive_test_clips_recallrrval_n_fprF2r)rr,rrr),rrxrrbTensorrrfloat32r param_groupsr zero_gradsqueezecatronesrr?rbackwardsteprrrrrrkrrrr8ranyrrrrr{rrr|getrr~)*rrrrrrrpositive_test_clipsrrruraccumulation_stepsaccumulated_samplesaccumulated_predictionsaccumulated_labelsstep_ndxdatar$rVy_gr neg_high_loss pos_high_losswrnrrkr val_step_ndxrrval_predictionsrtpfnrr/rrval_accr!r!r"rs "    6   $  $ "  (     " ""  zModel.train_model)r r rrr N)r)rrrrIrr)rr[r)rrr)r'r(r)rrrrr%rrrr rrr(rr*r!r!rr"r s6o    $ k r c Csddl}ddlm}ddl}||}||dd}tN}|tj |d|j j tj |d}|} td|dt|d  } | | Wdn1sVwYWddSWddS1snwYdS) zRConverts an ONNX version of an openwakeword model to the Tensorflow tflite format.rN)prepareCPUrxtf_modelz#### Saving tflite mode to 'rwb)ronnx_tf.backendrJ tensorflowloadtempfileTemporaryDirectory export_graphr"r#r$liteTFLiteConverterfrom_saved_modelconvertrropenwrite) onnx_model_pathrrrJtf onnx_modeltf_reptmp_dir converter tflite_modelfr!r!r"convert_onnx_to_tflite>s(           rc__main__z--training_configz/The path to the training config file (required)T)helptyperequiredz--generate_clipsz-Execute the synthetic data generation process store_trueFalseF)reactiondefaultrgz--augment_clipsz/Execute the synthetic data augmentation processz --overwritezNOverwrite existing openwakeword features when the --augment_clips flag is usedz --train_modelz"Execute the model training processrpiper_sample_generator_path)generate_samplesr&r%positive_train positive_testnegative_train negative_testcCs"g|] }t|D]}|jq qSr!)r"scandirr#)r.jr/r!r!r"r0s"r0 rir_paths!background_paths_duplication_ratebackground_pathsr cCsg|]}|jqSr!)r#r-r!r!r"r0z################################################## Generating positive clips for training ##################################################gffffff? n_samples target_phrasetts_batch_sizeg\(\?)g??g?cCg|] }tjdqSz.wavuuiduuid4hexr-r!r!r"r0r) text max_samplesr noise_scalesnoise_scale_ws length_scalesr&auto_reduce_batch_size file_namesz8Skipping generation of positive clips for training, as ~z already existz################################################## Generating positive clips for testing ################################################## n_samples_valr|)rrrrrrr&rz3Skipping generation of positive clips testing, as ~z################################################## Generating negative clips for training ##################################################custom_negative_phrasesr) input_textNinclude_partial_phraseinclude_input_wordscCr}r~rr-r!r!r"r0rz8Skipping generation of negative clips for training, as ~z################################################## Generating negative clips for testing ##################################################z7Skipping generation of negative clips for testing, as ~r-cCg|]}t|qSr!strr-r!r!r"r0r1z*.wavri. total_lengthi}izpositive_features_train.npycCrr!rr-r!r!r"r0r1augmentation_roundsaugmentation_batch_size)rrbackground_clip_paths RIR_pathscCrr!rr-r!r!r"r0r1cCrr!rr-r!r!r"r0r1cCrr!rr-r!r!r"r0r1z################################################## Computing openwakeword features for generated samples ##################################################rHgpur)n_total clip_duration output_filerxrznegative_features_train.npyzpositive_features_test.npyznegative_features_test.npyzVOpenwakeword features already exist, skipping data augmentation and feature generationrLi>r layer_sizei)r?r=rrrwrcsZjdksjdkr+ttfddtdjdD}|SS)zbSimple transformation function to ensure negative data is the appropriate shape for the model sizer cs$g|]}||ddfqSrr!r-nr$r!r"r0:s$zf..r)rrrarrayr8)r$r new_batchr!rr"rb6s  ,rbcCsi|]}|tqSr!)rb)r.rr!r!r" @rxrfeature_data_filespositiveadversarial_negativecCdd|DS)NcSg|]}dqSrSr!r-r!r!r"r0DrX..r!r$r!r!r"rWDrWcCr)NcSrrvr!r-r!r!r"r0FrXrr!rr!r!r"rWFrbatch_n_per_class) n_per_classdata_transform_funcslabel_transform_funcsc@seZdZddZddZdS) IterDatasetcCs ||_dSr generator)rrr!r!r"rUrzIterDataset.__init__cCs|jSrrrr!r!r"__iter__XszIterDataset.__iter__N)r'r(r)rrr!r!r!r"rTs r)r num_workersprefetch_factor#false_positive_validation_data_pathcCs g|] }t||tdqSrv)X_val_fpr=r-r!r!r"r0ds rrrtarget_false_positives_per_hour)rrrrrr)rr%r&rz.tflite)r)rbrrrrrr"sysrRrrrscipyrargparserryamlpathlibr openwakewordopenwakeword.datarrropenwakeword.utilsr r rr rcr'ArgumentParserparser add_argumentr parse_argsargsrQrYtraining_configreadLoaderconfigr#insertabspathrnexistsmkdirr$positive_train_output_dirpositive_test_output_dirnegative_train_output_dirnegative_test_output_dirfeature_save_dirrurwrrbackground_pathduplication_rateextendrsgenerate_clipsrlistdirn_current_samplesr8ry empty_cacheradversarial_textsrzrglobpositive_clipsduration_in_samplesr/iowavfilerandomrandintsrdatrrroundmedianabs overwritepositive_clips_trainpositive_clips_train_generatorpositive_clips_testpositive_clips_test_generatornegative_clips_trainnegative_clips_train_generatornegative_clips_testnegative_clips_test_generator cpu_countn_cpusrzrutilsrget_embedding_shaper=owwrbkeysdata_transformslabel_transformsrrbatch_generatorr?IterableDatasetr DataLoaderrrrrzerosrr/X_val_fp_labels TensorDatasetr X_val_pos X_val_neghstackr4labelsrrr rr(r!r!r!r"s2    )   "                    $"  $                      *.  [