86 lines
12 KiB
Plaintext
86 lines
12 KiB
Plaintext
|
o
|
|||
|
|
K�i+D ã @ s€ d dl mZ d dlZd dlZdejd< d dlmZ d dlZ e
|
|||
|
|
dd¡ d dl Z d dl
|
|||
|
|
Z
|
|||
|
|
d dlmZ d dlmZ d dlmZ d d lmZmZ d d
|
|||
|
|
lmZ d ZeejvrXej e¡ d d l m Z m Z m Z m Z d d
|
|||
|
|
l!m"Z" d dl#m$Z$ d dl%Z&d dl'm'Z' d dlmZ d dl(m)Z) d dl(m*Z* d dl+m,Z,m-Z- d dl.Z/d dl0Z/d dl1Z2d dl3Z4d dl5Z6d dl7m8Z9 d dl:m;Z; d dl1m<Z< d dl=m>Z> d dl?Z?d dl@Z@e2jAddd� d dlBmCZC d dlDmEZE d dlFmGZG d d lHmIZImJZJ d d lKmLZL d d lMmNZN d dlDmEZE d d lOmPZP d d lKmQZQ d d!lFmRZR d dlFmGZG d d"lHmIZI d d#lSmTZTmUZU d dlVZWd d$lXmYZY d d%lZm[Z[ d dl1m<Z< d dl:m;Z; d d&l\m]Z] d d'l^m_Z_ d d(lKm`Z` d d)lambZb d d*lDmcZc d dlFmGZG d d+lHmdZdmeZe d dlfZfd,d-„ Zgd.d/„ Zhd0d1„ Zid2d3„ Zjd4d5„ Zkd6d7„ Zld8d9„ Zmd:d;„ Znd<d=„ Zod>d?„ Zpd@dA„ ZqdBdC„ ZrdDdE„ ZsdFdG„ ZtdUdIdJ„ZudKdL„ ZvdMdN„ ZwdOdP„ ZxdQdR„ ZydSdT„ ZzdS )Vé NÚ0Z
|
|||
|
|
USE_PYGEOS)ÚMarkdownzdisplay.max_rows)Úconfigure_s3_access)Úmasking)Ú write_cog)Ú display_mapÚrgb)Úmostcommon_crsz/home/jovyan/easi-notebooks)Ú EasiDefaultsÚxarray_object_sizeÚnotebook_utilsÚunset_cachingproxy)Úload_s2l2a_with_offset)Úprogress)Údatetime)Ú enum_to_bool)Ú xr_reproject)ÚGeoBoxÚbox)Ú
|
|||
|
|
reductions)Úopts©Ú
|
|||
|
|
load_data_geoZbokehF)Zlogo)Úcalculate_indices)ÚRandomForestClassifier)Útrain_test_split)Úaccuracy_scoreÚclassification_report)Ú LabelEncoder)ÚPipeline)Ú
|
|||
|
|
SimpleImputer)ÚStandardScaler)Ú GridSearchCV)r )ÚPointÚPolygon)ÚCRS)ÚListedColormap)Ú FixedTicker)Ú merge_arrays)ÚPolynomialFeatures)ÚLinearRegression)ÚRandomForestRegressor)Úmean_squared_errorÚr2_scorec
|
|||
|
|
C s\ d}||||dœ}t | |¡}td|› �ƒ g d¢}||dddddœd œ}t| ||B ƒ} | S ©
|
|||
|
|
NZs2_l2a)ÚproductÚxÚyÚtimezMost common native CRS: )ÚredZnirÚscl©iöÿÿÿé
|
|||
|
|
Ú solar_dayé ©r0 r1 )Ú measurementsÚ
|
|||
|
|
output_crsÚ
|
|||
|
|
resolutionÚgroup_byÚ dask_chunks©r r Úprintr )
|
|||
|
|
ÚdcÚ
|
|||
|
|
date_rangeÚlongtitude_rangeÚlatitude_ranger/ ÚqueryÚ
|
|||
|
|
native_crsr: Ú load_paramsÚdata© rI ú5/home/x79/CSIROBoeingPhase5-Vietnam/new_import_ODC.pyÚ load_dataT s( ü ûþrK c s„ d}t | | ¡}t|ƒ t|jd jd ƒ |jd jd ‰ ‡ fdd„dD ƒ}t| | |ƒ}dd„ | jD ƒ}| | |¡ ¡ }|S )Nr4 Úqaé c ó g | ]}ˆ t |ƒ ‘qS rI ©Ústr©Ú.0Úi©Z flags_defrI rJ Ú
|
|||
|
|
<listcomp>u ó z mask_clean.<locals>.<listcomp>©é é é é c S ó g | ]}|d kr|‘qS ©r4 rI ©rR r0 rI rI rJ rU { rV © r Zdescribe_variable_flagsÚdisplayÚlocÚvaluesr Z data_varsÚwhereÚpersist©rH Z flag_nameZ flag_descZgood_pixel_flagsZgood_pixel_maskZdata_layer_namesÚresultrI rT rJ Ú
|
|||
|
|
mask_cleann s rg c C s€ g }|D ]!}| j |d�}|j |d�jdd�}|j |d�jdd�}| |¡ qtjdd„ |D ƒdd�}|jdd�}|jdd�}|S )N©r2 r2 )Údimc S s g | ]}|‘qS rI rI rQ rI rI rJ rU ˆ s z fill_nan.<locals>.<listcomp>)ÚselZbfillZffillÚappendÚxrÚconcat)ÚndviZ
|
|||
|
|
time_splitÚrsÚtimesÚtmpZfill_dsZ merged_ndviZfill_mrI rI rJ Úfill_nan� s rr c C s t | ƒ}|S ©Nr )Z
|
|||
|
|
train_pathÚtrainrI rI rJ Úload_train_dataŽ s ru c C s t |¡}t | ¡}||fS rs )Ú rioxarrayÚ
|
|||
|
|
open_rasterio)Zname_vhZname_vvÚdsvvÚdsvhrI rI rJ Ú load_sen1“ s
|
|||
|
|
|
|||
|
|
rz c C s i }| ¡ D ]X\}}d|d › �}z7|j|jj|jjdd�j}|j|jj|jjdd�j} |j|jj|jjdd�j}
|
|||
|
|
t || |
|
|||
|
|
f¡|jdœ||< W q t y^ } z
|
|||
|
|
t
|
|||
|
|
| ƒ W Y d } ~ qd } ~ ww |S )NZpoint_rM Únearest©r0 r1 Úmethod)rH Úlabel) Úiterrowsrj Úgeometryr0 r1 rb ÚnpÚ concatenateZHT_codeÚ Exceptionr@ ) rt Ú average_ndviry rx Zloaded_datasetsÚidxÚpointÚkeyZ ndvi_dataÚvh_dataÚvv_dataÚerI rI rJ Úget_data_sen1_and_sen2™ s þ€þr‹ c sÌ t ƒ }| jj}| ‡ fdd„|D ƒ¡}g }g }g }| ¡ D ] \} }
|
|||
|
|
| |
|
|||
|
|
¡ q tt|ƒƒD ]} || d urE| || d ¡ | || ¡ q-t||ddd�\} }
|
|||
|
|
}}t|
|
|||
|
|
|ddd�\}}}}| |||||fS )Nc s g | ]}ˆ | ‘qS rI rI )rR r~ ©Ú
|
|||
|
|
label_mappingrI rJ rU ° s z$split_train_data.<locals>.<listcomp>rH gš™™™™™Ù?é* )Z test_sizeÚ random_stateg à?) r Z Hientrangrb Z
|
|||
|
|
fit_transformÚitemsrk ÚrangeÚlenr )rt r� ÚdatasetsZ
|
|||
|
|
label_encoderÚlabelsZnumeric_labelsÚXZx_newZlb_newÚkÚvrS ÚX_trainZX_tempÚy_trainZy_tempÚX_valZX_testÚy_valZy_testrI rŒ rJ Úsplit_train_data« s €rœ c C sš t ddd�}tdtƒ fd|fgƒ}g d¢g d¢dd gd
|
|||
|
|
œ}t||d d dd
|
|||
|
|
�}| | |¡ |j}td|ƒ | |¡} t|| ƒ}
|
|||
|
|
tdt |
|
|||
|
|
dƒd › d�ƒ |S ©NrŽ éÿÿÿÿ)r� Ún_jobsZscalerZ
|
|||
|
|
classifier)éd i, iô i¼ iè )r[ é r6 é é ZginiÚentropy)Zclassifier__n_estimatorsZclassifier__max_depthZclassifier__criterionrZ Úaccuracy)ÚcvZscoringrŸ zBest Parameters:z
|
|||
|
|
Accuracy: rX r z %©
|
|||
|
|
r r r! r" ÚfitZ best_params_r@ Úpredictr Úround) r˜ rš r™ r› Ú
|
|||
|
|
base_modelÚpipelineÚ
|
|||
|
|
param_gridÚ grid_searchÚ best_paramsÚy_predr¥ rI rI rJ Ú
|
|||
|
|
train_with_rf¿ s" ýý
|
|||
|
|
|
|||
|
|
|
|||
|
|
r± c C s< d}t j |¡s
|
|||
|
|
t |¡ t |t j || ¡¡ tdƒ d S )NZ model_trainzDone!)ÚosÚpathÚexistsÚmkdirÚjoblibÚdumpÚjoinr@ )Z name_filer® Zdir_save_modelrI rI rJ Ú
|
|||
|
|
save_modelã s
|
|||
|
|
|
|||
|
|
r¹ c C sô g }t |jd ƒD ]1}|j|d�j}|j|jj| dd�j}|j|jj| dd�j} tj||| fdd�}
|
|||
|
|
| |
|
|||
|
|
j ¡ q |
|
|||
|
|
|¡} | |jjd |j jd ¡} t
|
|||
|
|
j| dd�}
|
|||
|
|
|
|
|||
|
|
j |¡}
|
|||
|
|
|j j}|jj}t
|
|||
|
|
j|
|
|||
|
|
||d œd
|
|||
|
|
d gd �}|j |jj¡}|S )
|
|||
|
|
NrM )r1 r{ )r1 r} r )Úaxis)r1 r0 )Údimsr9 r1 r0 )Úcoordsr» )r‘ ÚshapeZiselrb rj r1 r� r‚ ÚextendÚTr© Úreshaper0 rl Z DataArrayÚrioZ write_crsÚcrs)ÚmodelZdata_crsrn ÚvhÚvvZ data_predictrS Zndvi_tmprˆ r‰ Zall_tmpr° Z final_labelZfinal_xarray_saveZx_valuesZy_valuesÚ
|
|||
|
|
data_arrayrI rI rJ r© ë s&
|
|||
|
|
þr© c C sv t | ¡}| |jj¡}t|jjd jj ƒ}dd„ |D ƒ}d|gdœg}|jj
|
|||
|
|
||jjdd�}| |dkt dƒ¡}|S ) Nr c S s g | ]\}}||f‘qS rI rI )rR r0 r1 rI rI rJ rU rV z%cut_according_shp.<locals>.<listcomp>r$ )ÚtypeÚ coordinatesF)ÚdropÚnan)
|
|||
|
|
ÚgpdÚ read_fileZto_crsrÁ r Úlistr€ rb Zexteriorr¼ Úcliprc Úfloat)Z
|
|||
|
|
thuanhoa_pathr„ rÆ ÚgdfZpolygon_coordsZpolygon_coordinatesZ
|
|||
|
|
geometriesZ
|
|||
|
|
region_resultrI rI rJ Úcut_according_shp s
|
|||
|
|
þÿrÑ c C s† t j| dd�}|jj}|jj}t |¡}|d d d }|d d d } i }
|
|||
|
|
| ¡ D ]–\} } td| › �ƒ g }
|
|||
|
|
t t
|
|||
|
|
|ƒƒD ]{}|| }|| }| ||¡}zY|j
|
|||
|
|
|gd¡}|| d v r‘|| krl| |t| ƒktj¡}n4||kr…| |t| ƒktj¡}| |dktj¡}n| |t|ƒktj¡}ntj|jd d …d d …d d …f< |
|
|||
|
|
|¡ W q= ty¸ } zW Y d }~q=d }~ww |
|
|||
|
|
| |
|
|||
|
|
i¡ q*|
|
|||
|
|
S ) Nz EPSG:9209)r ZTQrH r ZPNNzprocess g @)rË rÌ r€ rb Ztenchurv rw r� r@ r‘ r’ ÚgetrÁ rÎ rc rÏ r� rÊ rk rƒ Úupdate)ZKD_pathZKetQuaPhanLoaiDatZCODE_MAPÚHT_MAPrÐ Úpolygonr~ Z ouput_imageZcode_tqZcode_pnnrf r‡ rb Z
|
|||
|
|
array_listrS ÚpoÚlbZcode_lbÚqrrŠ rI rI rJ Úcompare s>
|
|||
|
|
€ÿrÙ c C sl d}t j |¡s
|
|||
|
|
t |¡ | ¡ D ]"\}}t|tjd�}|j |› d|› d�¡ t
|
|||
|
|
d|› d|› d�ƒ qd S )NzThuanHoa/KetQua©Znodataú/ú.tifúsave © r² r³ r´ rµ r� r( r� rÊ rÁ Ú to_rasterr@ )rf rÔ Ú save_pathr– r— ro rI rI rJ Ú save_result7 s
|
|||
|
|
ýrá c C s\ |\}}| j d|||ddgdddddœdd d
|