__pycache__/train_module.cpython-310.pyc

o
¤™>iEQã@sŠdZddlZddlZddlZddlmZddl	m
Z
ddlmZm
Z
ddlmZddlmZddlmZdd	lmZddlZdd
lmZddlZddlZddlZddlZddlmZe d¡z%ddlZddl m!Z!ddl"m!m#Z$ddl%m&Z&dd
l'm(Z(m)Z)dZ*Wne+y“dZ*e,dƒYnwGdd„de!j-ƒZ.ddl/Z/ddl0m1Z1ddl2m3Z4gd¢ddddddddddddddfdd„Z5dS) zy
Training module for land classification using Sentinel-2 and Sentinel-1 data
from Microsoft Planetary Computer STAC API
éN)Útrain_test_split)ÚLabelEncoder)Úclassification_reportÚconfusion_matrix)ÚRandomForestClassifier)ÚDecisionTreeClassifier)ÚSVC)Ú
XGBClassifier)Údatetime)ÚPathÚignore)Ú
TensorDatasetÚ
DataLoaderTFz8Warning: PyTorch not available. CNN model will not work.cs4eZdZ‡fdd„Zdd„Zdd„Zdd„Z‡ZS)	Ú
CNNClassifiercsôtt|ƒ ¡||_||_|dkr7d|_t |d¡|_t 	d¡|_
t dd¡|_t 	d¡|_t d|¡|_
dSd|_tjdd	d
dd�|_tjdd
�|_tjd	dd
dd�|_tjdd
�|_|ddd}t |d¡|_t 	d¡|_t d|¡|_dS)NéFé@g333333Ó?é€gà?Téé é)Zin_channelsZout_channelsÚkernel_sizeÚpaddingé)r)ÚsuperrÚ__init__Ú
n_featuresÚ	n_classesÚuse_convÚnnZLinearÚfc1ZDropoutÚdropout1Úfc2Údropout2Úfc3ZConv1dÚconv1Z	MaxPool1dÚpool1Úconv2Úpool2Údropout)ÚselfrrZconv_output_size©Ú	__class__©ú3/home/x79/CSIROBoeingPhase5-Vietnam/train_module.pyr's&zCNNClassifier.__init__cCsæ|jrFt|jƒdkr| d¡}t | |¡¡}| |¡}t | |¡¡}| 	|¡}| 
| d¡d¡}t | |¡¡}| 
|¡}| |¡}|St|jƒdkrR| d¡}t | |¡¡}| |¡}t | |¡¡}| |¡}| |¡}|S)Nrrréÿÿÿÿr)rÚlenÚshapeÚ	unsqueezeÚFZrelur$r%r&r'ÚviewÚsizerr(r!Úsqueezer r"r#)r)Úxr,r,r-ÚforwardEs(


ù


zCNNClassifier.forwardcCs¬| ¡t ¡�Dt|tjƒrt |¡}|js$t|j	ƒdkr$| 
d¡}n|jr3t|j	ƒdkr3| d¡}||ƒ}t |d¡\}}| 
¡ ¡WdƒS1sOwYdS)z!Scikit-learn style predict methodrrrN)ÚevalÚtorchZno_gradÚ
isinstanceÚnpÚndarrayÚFloatTensorrr/r0r5r1ÚmaxÚcpuÚnumpy)r)ÚXÚoutputsÚ_Z	predictedr,r,r-Úpredict^s


$özCNNClassifier.predictcCs0| |¡}t|tjƒr| ¡ ¡}t ||k¡S)zScikit-learn style score method)rDr:r9ZTensorr?r@r;Úmean)r)rAÚyÚpredictionsr,r,r-Úscorems
zCNNClassifier.score)Ú__name__Ú
__module__Ú__qualname__rr7rDrHÚ
__classcell__r,r,r*r-r&s
r)ÚClient)Úload)gffffffZ@gš™™™™™"@gÍÌÌÌÌŒZ@gš™™™™™#@z2023-03-01/2023-05-31éééz1train/ST_training data_updated_1130points_new.shpÚxgboostédgš™™™™™¹?cS
s˜
d�‡fdd„	}‡fdd„}�zõ|dur#t ¡ d¡}d|›d|›d	�}td
ƒ}|jdd�|›d|›d|›d|›d|›�	}t | ¡¡ ¡}|d
|›d	�}d}d}|rœ| 	¡rœ|d|j
›d�dƒzt |¡}|d}|d}|dt
|ƒ›d�dƒWnty›}z|dt|ƒ›d�dƒd}WYd}~nd}~ww|du�r³|ddƒ|ddƒt d¡}|ƒ|ddƒ|jdg||dd |iid!�}t| ¡ƒ}|ƒt
|ƒ|kræt
|ƒ|}|dd|…d|…}|d"t
|ƒ›d#�d$ƒ|d%d&ƒd'd(„|Dƒ}t|gd)¢d*||tjd+d,�}| d-d.d/d)œ¡}|ƒ|d0d1ƒ|jd2g||d3�}t| ¡ƒ} t
| ƒ|k�rCt
| ƒ|}| dd|…d|…} |d"t
| ƒ›d4�d5ƒ|d6d7ƒd8d(„| Dƒ} t| d9d:gd*||tjd+d,�}!dt |!d9 |!d9d;k¡¡|!d<<dt |!d: |!d:d;k¡¡|!d=<|ƒ|d>dƒ|d.|d-|d.|d-d?}"|d/ gd@¢¡}#|" |#¡}$|$jdAdB�}%|dCdDƒt |¡}&|&j d*k�rÐ|& !d*¡}&d}'dED]
}(|(|&j"v�rà|(}'n�qÔ|'du�rñt#dFt|&j"ƒ›�ƒ‚|dGdHƒg}g}|& $¡D]X\})}*|*j%}+|+j&},|+j'}-|*|'}.zA|%j(|,|-dIdJ�j)}/|!d=j(|,|-dIdJ�jdAdB�j)}0|!d<j(|,|-dIdJ�jdAdB�j)}1|/|0|1g}2t *|2¡ +¡�sN| ,|2¡| ,|.¡W�qþY�qþt -|¡}t -|¡}|ƒ|dKt
|ƒ›dL�dMƒ|�r³|dNdOƒz|||||t ¡ .¡dPœ}3t /|3|¡|dQ|j
›�dRƒWnt�y²}z|dSt|ƒ›�dRƒWYd}~nd}~wwt0ƒ}4|4 1|¡}5t2||5dTdU|5dV�\}6}7}8}9|dW| 3¡›dX�dRƒ|
�rØdYndZ}:|d[k�ròt4|||	|
�rè|:ndZd\dUd]d;d^�};�n|d_k�rt5||dUd`d;da�};�n|dbk�rt6|dUdc�};nù|ddk�rt7dedUd+df�};nì|dgk�rÿt8�s't9dhƒ‚|6j:di}<t
t ;|8¡ƒ}=t< =t<j> ?¡�r@|
�r@djndZ¡}:|dk|:›d�dRƒt@|<|=ƒ A|:¡};t< B|6¡ Cdi¡}>t< D|8¡}?t< B|7¡ Cdi¡}@t< D|9¡}AtE|>|?ƒ}BtF|Bdlddm�}CtG H¡}DtIjJ|; K¡dndo�}E|dpdqƒtLd|drƒ}F|; M¡tN|FƒD]Y}Gds}H|CD]+\}I}J|I A|:¡|J A|:¡}I}J|E O¡|;|Iƒ}K|D|K|Jƒ}L|L P¡|E Q¡|H|L R¡7}H�q¡|Gdidd;k�ró|Ht
|Cƒ}M|dt|Gdi›du|F›dv|Mdw›�dq|G|Fdƒ�q›|; S¡};t|:ƒ|;_Tnt#dx|›dy�ƒ‚|dgk�r|; U|6|8¡|dzd{ƒ|dgk�r)|; V|6|8¡}N|; V|7|9¡}On|; V|6|8¡}N|; V|7|9¡}O|d|d}ƒtWjXtWjY Z|¡dd�t /|;|4d~œ|¡idt ¡ .¡“d€d�“d‚dd2g“dgdƒ¢“d„t
|6ƒ“d…t
|7ƒ“d†t[|Nƒ“d‡t[|Oƒ“dˆ|“d‰|d[k�r‚|:ndZ“dŠ|d‹v�rŒ|nd“dŒ|dgk�r–|nd“d�|d[k�r |	nd“dŽ|dgk�r¯tLd|drƒnd“d�|6j:di“d�t
t ;|8¡ƒ“d‘|“||d’œ¥}P| \d	d“¡}Qt|Qd”ƒ�}Rt]j/|P|Rdrd•�Wdƒn	1�sçwY|d–d—ƒd||Q|N|Ot
|6ƒt
|7ƒ|4j^ _¡d˜œWSt`�y(}z|d™t|ƒ›�d`ƒd+t|ƒddšœWYd}~Sd}~wt�yK}z|d›t|ƒ›�d`ƒd+t|ƒdœœWYd}~Sd}~ww)ža}
    Train a land classification model using Sentinel-2 and Sentinel-1 data
    
    Args:
        bbox: [min_lon, min_lat, max_lon, max_lat]
        time_range: "YYYY-MM-DD/YYYY-MM-DD"
        max_scenes: maximum number of scenes to load
        cloud_cover: maximum cloud cover percentage
        resolution: resolution in meters (e.g., 20)
        training_shapefile: path to training shapefile
        n_estimators: number of trees for XGBoost
        max_depth: maximum tree depth
        learning_rate: learning rate for XGBoost
        use_gpu: whether to use GPU for training
        output_model_path: path to save trained model (auto-generated if None)
        status_callback: Optional callback function to report progress
        cancel_check: Optional function that returns True if training should be cancelled
    
    Returns:
        Dictionary containing training results
    Ncs:ˆrzˆ||ƒWn
tyˆ|ƒYnwt|ƒdS)zHelper to update statusN)Ú	TypeErrorÚprint)ÚmessageÚprogress)Ústatus_callbackr,r-Ú
update_status¡sÿz"train_model.<locals>.update_statuscsˆr	ˆƒrtdƒ‚dSdS)z%Check if training should be cancelledzTraining cancelled by userN)ÚInterruptedErrorr,)Úcancel_checkr,r-Úcheck_cancellation«s
ÿz'train_model.<locals>.check_cancellationz
%Y%m%d_%H%M%Szmodel_train/model_rCz.joblibÚ
dataset_cacheT)Úexist_okZtraining_data_u!ðŸ“¦ Loading cached dataset from z...éÚfeaturesÚlabelsuâœ… Loaded z1 samples from cache (skipped satellite download!)é2uâš ï¸� Cache load failed: z, downloading fresh data...é
u?ðŸ“¡ Cache not found or disabled, downloading satellite data...z-Connecting to Microsoft Planetary Computer...rOz3https://planetarycomputer.microsoft.com/api/stac/v1z"Searching for Sentinel-2 scenes...zsentinel-2-l2azeo:cloud_coverÚlt)ÚcollectionsÚbboxr
ÚqueryzFound z Sentinel-2 scenesrQzLoading Sentinel-2 data...écSóg|]}t |¡‘qSr,©Úplanetary_computerÚsign©Ú.0Úitemr,r,r-Ú
<listcomp>íóztrain_model.<locals>.<listcomp>)ÚB04ÚB08ÚSCLz
EPSG:32648F)ZbandsÚcrsÚ
resolutionrfZ	patch_urlZ
fail_on_errorÚredÚnirÚsclz"Searching for Sentinel-1 scenes...é#zsentinel-1-rtc)rerfr
z Sentinel-1 scenesé(zLoading Sentinel-1 data...é-cSrir,rjrmr,r,r-rp
rqÚvvÚvhrZvv_dbZvh_dbzCalculating NDVI...g:Œ0âŽyE>)rrré	rcÚtime)ÚdimzLoading training data...é7)ZHT_codeZMa_LUZLU2022Z	HientrangÚclassZClassÚCLASSz2Cannot find label column in shapefile. Available: z+Extracting features from training points...é<Únearest)r6rFÚmethodz
Extracted z valid training sampleséFu.ðŸ’¾ Saving dataset to cache for future use...éH)r`rarfÚ
time_rangervÚ	timestampuâœ… Cached to éKuâš ï¸� Cache save failed: gš™™™™™É?é*)Z	test_sizeÚrandom_stateZstratifyz	Training z	 model...zcuda:0r?rRÚhistZmlogloss)Ún_estimatorsÚ	max_depthÚ
learning_rateÚdeviceZtree_methodrŽZeval_metricÚ	verbosityÚ
random_forestr.)r�r‘rŽÚn_jobsÚverboseZ
decision_tree)r‘rŽZsvmZrbf)ZkernelrŽr—Úcnnz7PyTorch is required for CNN. Install: pip install torchrÚcudazBuilding CNN model on r)Ú
batch_sizeÚshufflegü©ñÒMbP?)Úlrz"Training CNN model with PyTorch...éPrgz
CNN Epoch ú/z, Loss: z.4fzUnknown model type: z9. Choose: xgboost, random_forest, decision_tree, svm, cnnzEvaluating model...éZzSaving model...é_)ÚmodelÚ
label_encoderr‹Zdata_sourcez!Microsoft Planetary Computer STACre)Z	NDVI_meanZ
VH_dB_meanZ
VV_dB_meanÚtraining_samplesÚtesting_samplesÚtrain_accuracyÚ
test_accuracyÚ
model_typer“r�)rRr•r˜r‘r’Z
cnn_epochsrrrf)rŠrvz
_info.jsonÚw)ÚindentzTraining complete!rS)ÚsuccessÚ
model_pathÚ	info_pathr¥r¦r£r¤ÚclasseszCancelled: )rªÚerrorÚ	cancelledzError: )rªr®)N)ar
ÚnowÚstrftimerÚmkdirÚhashlibÚmd5ÚencodeÚ	hexdigestÚexistsÚnameÚjoblibrNr/Ú	ExceptionÚstrrMÚopenÚsearchÚlistZitem_collectionÚ	stac_loadrkrlÚrenamer;Úlog10ÚwhereÚisinrEÚgpdÚ	read_fileruZto_crsÚcolumnsÚ
ValueErrorZiterrowsZgeometryr6rFZselÚvaluesÚisnanÚanyÚappendÚarrayÚ	isoformatÚdumprZ
fit_transformrÚupperr	rrrÚPYTORCH_AVAILABLEÚImportErrorr0Úuniquer9r“r™Zis_availablerÚtor=r1Z
LongTensorr
rrZCrossEntropyLossÚoptimZAdamÚ
parametersÚminZtrainÚrangeZ	zero_gradÚbackwardÚstepror?Zdevice_usedÚfitrHÚosÚmakedirsÚpathÚdirnameÚfloatÚreplaceÚjsonZclasses_ÚtolistrZ)SrfrŠÚ
max_scenesÚcloud_coverrvÚtraining_shapefiler§r�r‘r’Úuse_gpuÚ	use_cacheZoutput_model_pathrXr[rYr\r‹Ú	cache_dirZcache_paramsÚ	cache_keyÚ
cache_filer`raZcached_dataÚeÚcatalogZquery_s2Zitems_s2rÙZds_s2Zquery_s1Zitems_s1Zds_s1ÚndviÚ
cloud_maskZndvi_maskedZ	ndvi_meanZ	train_gdfZlabel_columnÚcolÚidxÚrowZpointZx_coordZy_coordÚlabelZndvi_valZvh_valZvv_valZfeature_vecZ
cache_datar¢Zlabels_encodedZX_trainZX_testZy_trainZy_testr“r¡rrZX_train_tensorZy_train_tensorZ
X_test_tensorZ
y_test_tensorZ
train_datasetZtrain_loaderZ	criterionZ	optimizerZepochsZepochZ
epoch_lossZbatch_XZbatch_yrBZlossZavg_lossZtrain_scoreZ
test_scoreÚinfor¬Úfr,)r[rXr-Útrain_modelzs>'
 
€þ