464 lines
14 KiB
Plaintext
464 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"group\n",
|
|
"1 126406.875\n",
|
|
"2 105338.625\n",
|
|
"3 84267.750\n",
|
|
"4 63212.625\n",
|
|
"5 42147.000\n",
|
|
"6 21084.000\n",
|
|
"7 0.000\n",
|
|
"8 543558.750\n",
|
|
"9 515849.250\n",
|
|
"10 494773.125\n",
|
|
"11 473647.125\n",
|
|
"12 452282.250\n",
|
|
"13 431227.125\n",
|
|
"14 410216.625\n",
|
|
"15 388972.500\n",
|
|
"16 367639.125\n",
|
|
"17 346610.250\n",
|
|
"18 325187.625\n",
|
|
"19 303883.125\n",
|
|
"20 282849.000\n",
|
|
"21 261846.375\n",
|
|
"22 240363.375\n",
|
|
"23 219006.375\n",
|
|
"24 197497.125\n",
|
|
"25 176457.750\n",
|
|
"26 148561.875\n",
|
|
"27 127115.625\n",
|
|
"28 106081.500\n",
|
|
"29 84666.750\n",
|
|
"30 63490.875\n",
|
|
"31 42406.875\n",
|
|
"32 21404.250\n",
|
|
"33 0.000\n",
|
|
"34 487780.125\n",
|
|
"35 466738.125\n",
|
|
"36 445693.500\n",
|
|
"37 422475.375\n",
|
|
"38 401428.125\n",
|
|
"39 380341.500\n",
|
|
"40 359289.000\n",
|
|
"41 338241.750\n",
|
|
"42 317212.875\n",
|
|
"43 296205.000\n",
|
|
"44 275139.375\n",
|
|
"45 254113.125\n",
|
|
"46 248435.250\n",
|
|
"47 227380.125\n",
|
|
"48 206338.125\n",
|
|
"49 185317.125\n",
|
|
"50 164227.875\n",
|
|
"51 143180.625\n",
|
|
"52 122096.625\n",
|
|
"53 101028.375\n",
|
|
"54 84273.000\n",
|
|
"55 63210.000\n",
|
|
"56 42152.250\n",
|
|
"57 21063.000\n",
|
|
"58 0.000\n",
|
|
"59 269338.125\n",
|
|
"60 247128.000\n",
|
|
"61 226057.125\n",
|
|
"62 204975.750\n",
|
|
"63 183962.625\n",
|
|
"64 162873.375\n",
|
|
"65 141862.875\n",
|
|
"66 120849.750\n",
|
|
"67 99789.375\n",
|
|
"68 78705.375\n",
|
|
"69 57647.625\n",
|
|
"70 36639.750\n",
|
|
"71 22120.875\n",
|
|
"72 0.000\n",
|
|
"73 6370203.000\n",
|
|
"74 6349189.875\n",
|
|
"75 6328179.375\n",
|
|
"76 6307153.125\n",
|
|
"77 6286082.250\n",
|
|
"78 6273143.625\n",
|
|
"79 6251915.250\n",
|
|
"80 6230881.125\n",
|
|
"81 6209854.875\n",
|
|
"82 6188807.625\n",
|
|
"83 6167731.500\n",
|
|
"84 6146681.625\n",
|
|
"85 6125613.375\n",
|
|
"86 6104595.000\n",
|
|
"87 6083526.750\n",
|
|
"88 6062487.375\n",
|
|
"89 6041437.500\n",
|
|
"90 6020382.375\n",
|
|
"91 5999327.250\n",
|
|
"92 5978261.625\n",
|
|
"93 5957224.875\n",
|
|
"94 5936148.750\n",
|
|
"95 5915075.250\n",
|
|
"96 5894028.000\n",
|
|
"97 5876582.250\n",
|
|
"98 5855558.625\n",
|
|
"99 5834516.625\n",
|
|
"100 5813498.250\n",
|
|
"101 5792445.750\n",
|
|
"102 5771398.500\n",
|
|
"103 5750351.250\n",
|
|
"104 5729280.375\n",
|
|
"105 5708217.375\n",
|
|
"106 5687199.000\n",
|
|
"107 5666128.125\n",
|
|
"108 5645044.125\n",
|
|
"109 5623970.625\n",
|
|
"110 5602952.250\n",
|
|
"111 5584931.625\n",
|
|
"112 5563905.375\n",
|
|
"113 5542900.125\n",
|
|
"114 5521863.375\n",
|
|
"115 5500829.250\n",
|
|
"116 5479803.000\n",
|
|
"117 5458782.000\n",
|
|
"118 5437724.250\n",
|
|
"119 5416650.750\n",
|
|
"120 5395624.500\n",
|
|
"121 5374606.125\n",
|
|
"122 5353548.375\n",
|
|
"123 5332482.750\n",
|
|
"124 5311404.000\n",
|
|
"125 5290401.375\n",
|
|
"126 5269317.375\n",
|
|
"127 5248233.375\n",
|
|
"128 5227136.250\n",
|
|
"129 5206128.375\n",
|
|
"130 5185041.750\n",
|
|
"131 5164028.625\n",
|
|
"132 5143005.000\n",
|
|
"133 5121944.625\n",
|
|
"134 5100876.375\n",
|
|
"135 5079855.375\n",
|
|
"136 5058768.750\n",
|
|
"137 5037755.625\n",
|
|
"138 5016705.750\n",
|
|
"139 4995655.875\n",
|
|
"140 4974577.125\n",
|
|
"141 4953535.125\n",
|
|
"142 4932451.125\n",
|
|
"143 4911361.875\n",
|
|
"144 4890270.000\n",
|
|
"145 4869212.250\n",
|
|
"146 4859124.375\n",
|
|
"147 4838116.500\n",
|
|
"148 4817058.750\n",
|
|
"149 4795974.750\n",
|
|
"150 4774901.250\n",
|
|
"151 4753851.375\n",
|
|
"152 4732767.375\n",
|
|
"153 4711722.750\n",
|
|
"154 4690675.500\n",
|
|
"155 4669581.000\n",
|
|
"156 4648567.875\n",
|
|
"157 4627546.875\n",
|
|
"158 4606497.000\n",
|
|
"159 4585439.250\n",
|
|
"160 4564376.250\n",
|
|
"161 4543334.250\n",
|
|
"162 4522300.125\n",
|
|
"163 4501279.125\n",
|
|
"164 4480268.625\n",
|
|
"165 4459205.625\n",
|
|
"166 4438161.000\n",
|
|
"167 4417087.500\n",
|
|
"168 4396006.125\n",
|
|
"169 4375003.500\n",
|
|
"170 4353919.500\n",
|
|
"171 4332861.750\n",
|
|
"172 4311827.625\n",
|
|
"173 4290748.875\n",
|
|
"174 4269743.625\n",
|
|
"175 4248672.750\n",
|
|
"176 4227609.750\n",
|
|
"177 4206583.500\n",
|
|
"178 4185528.375\n",
|
|
"179 4164470.625\n",
|
|
"180 4143441.750\n",
|
|
"181 4122436.500\n",
|
|
"182 4102210.875\n",
|
|
"183 4081184.625\n",
|
|
"184 4060168.875\n",
|
|
"185 4039134.750\n",
|
|
"186 4018103.250\n",
|
|
"187 3997090.125\n",
|
|
"188 3976035.000\n",
|
|
"189 3955000.875\n",
|
|
"190 3933987.750\n",
|
|
"191 3912953.625\n",
|
|
"192 3891861.750\n",
|
|
"193 3870817.125\n",
|
|
"194 3849806.625\n",
|
|
"195 3828790.875\n",
|
|
"196 3807699.000\n",
|
|
"197 3786617.625\n",
|
|
"198 3765612.375\n",
|
|
"199 3744583.500\n",
|
|
"200 3723544.125\n",
|
|
"201 3702454.875\n",
|
|
"202 3681431.250\n",
|
|
"203 3660420.750\n",
|
|
"204 3639384.000\n",
|
|
"205 3618352.500\n",
|
|
"206 3598822.500\n",
|
|
"207 3577777.875\n",
|
|
"208 3556735.875\n",
|
|
"209 3535722.750\n",
|
|
"210 3514667.625\n",
|
|
"211 3493659.750\n",
|
|
"212 3472607.250\n",
|
|
"213 3451575.750\n",
|
|
"214 3435453.000\n",
|
|
"215 3414371.625\n",
|
|
"216 3393363.750\n",
|
|
"217 3372355.875\n",
|
|
"218 3351353.250\n",
|
|
"219 3330292.875\n",
|
|
"220 3309214.125\n",
|
|
"221 3288211.500\n",
|
|
"222 3267143.250\n",
|
|
"223 3246082.875\n",
|
|
"224 3225043.500\n",
|
|
"225 3204001.500\n",
|
|
"226 3182972.625\n",
|
|
"227 3161928.000\n",
|
|
"228 3140880.750\n",
|
|
"229 3119830.875\n",
|
|
"230 3098770.500\n",
|
|
"231 3077702.250\n",
|
|
"232 3056618.250\n",
|
|
"233 3035547.375\n",
|
|
"234 3014460.750\n",
|
|
"235 2993387.250\n",
|
|
"236 2972316.375\n",
|
|
"237 2951308.500\n",
|
|
"238 2930237.625\n",
|
|
"239 2909221.875\n",
|
|
"240 2888130.000\n",
|
|
"241 2867038.125\n",
|
|
"242 2845948.875\n",
|
|
"243 2824938.375\n",
|
|
"244 2803925.250\n",
|
|
"245 2782846.500\n",
|
|
"246 2761799.250\n",
|
|
"247 2740757.250\n",
|
|
"248 2718371.250\n",
|
|
"249 2697321.375\n",
|
|
"250 2676255.750\n",
|
|
"251 2655213.750\n",
|
|
"252 2634129.750\n",
|
|
"253 2613082.500\n",
|
|
"254 2592051.000\n",
|
|
"255 2570990.625\n",
|
|
"256 2549946.000\n",
|
|
"257 2528911.875\n",
|
|
"258 2507904.000\n",
|
|
"259 2486848.875\n",
|
|
"260 2465801.625\n",
|
|
"261 2444770.125\n",
|
|
"262 2423678.250\n",
|
|
"263 2402675.625\n",
|
|
"264 2381646.750\n",
|
|
"265 2360596.875\n",
|
|
"266 2339568.000\n",
|
|
"267 2318484.000\n",
|
|
"268 2297463.000\n",
|
|
"269 2276457.750\n",
|
|
"270 2255407.875\n",
|
|
"271 2234400.000\n",
|
|
"272 2213397.375\n",
|
|
"273 2192350.125\n",
|
|
"274 2171255.625\n",
|
|
"275 2150216.250\n",
|
|
"276 2129142.750\n",
|
|
"277 2108082.375\n",
|
|
"278 2087011.500\n",
|
|
"279 2065966.875\n",
|
|
"280 2044859.250\n",
|
|
"281 2023798.875\n",
|
|
"282 2002712.250\n",
|
|
"283 1981665.000\n",
|
|
"284 1963833.375\n",
|
|
"285 1942799.250\n",
|
|
"286 1921736.250\n",
|
|
"287 1900660.125\n",
|
|
"288 1879594.500\n",
|
|
"289 1858523.625\n",
|
|
"290 1837479.000\n",
|
|
"291 1816450.125\n",
|
|
"292 1795360.875\n",
|
|
"293 1783123.125\n",
|
|
"294 1782676.875\n",
|
|
"295 1761671.625\n",
|
|
"296 1740627.000\n",
|
|
"297 1719606.000\n",
|
|
"298 1698585.000\n",
|
|
"299 1677548.250\n",
|
|
"300 1656519.375\n",
|
|
"301 1635477.375\n",
|
|
"302 1614451.125\n",
|
|
"303 1593359.250\n",
|
|
"304 1572325.125\n",
|
|
"305 1551285.750\n",
|
|
"306 1530228.000\n",
|
|
"307 1509146.625\n",
|
|
"308 1488109.875\n",
|
|
"309 1467091.500\n",
|
|
"310 1446041.625\n",
|
|
"311 1424999.625\n",
|
|
"312 1403976.000\n",
|
|
"313 1383823.875\n",
|
|
"314 1362737.250\n",
|
|
"315 1341648.000\n",
|
|
"316 1320553.500\n",
|
|
"317 1299498.375\n",
|
|
"318 1278482.625\n",
|
|
"319 1257414.375\n",
|
|
"320 1236343.500\n",
|
|
"321 1215272.625\n",
|
|
"322 1194217.500\n",
|
|
"323 1173133.500\n",
|
|
"324 1152054.750\n",
|
|
"325 1130997.000\n",
|
|
"326 1109913.000\n",
|
|
"327 1088844.750\n",
|
|
"328 1067755.500\n",
|
|
"329 1046729.250\n",
|
|
"330 1025640.000\n",
|
|
"331 1004556.000\n",
|
|
"332 983487.750\n",
|
|
"333 962456.250\n",
|
|
"334 941445.750\n",
|
|
"335 920361.750\n",
|
|
"336 899277.750\n",
|
|
"337 878175.375\n",
|
|
"338 857128.125\n",
|
|
"339 836122.875\n",
|
|
"340 815091.375\n",
|
|
"341 794044.125\n",
|
|
"342 773036.250\n",
|
|
"343 752007.375\n",
|
|
"344 731004.750\n",
|
|
"345 709957.500\n",
|
|
"346 688949.625\n",
|
|
"347 673727.250\n",
|
|
"348 652666.875\n",
|
|
"349 631632.750\n",
|
|
"350 610598.625\n",
|
|
"351 589509.375\n",
|
|
"352 568480.500\n",
|
|
"353 547407.000\n",
|
|
"354 526375.500\n",
|
|
"355 505336.125\n",
|
|
"356 484273.125\n",
|
|
"357 463199.625\n",
|
|
"358 442139.250\n",
|
|
"359 421086.750\n",
|
|
"360 400008.000\n",
|
|
"361 378973.875\n",
|
|
"362 357895.125\n",
|
|
"363 336882.000\n",
|
|
"364 315819.000\n",
|
|
"365 294771.750\n",
|
|
"366 273708.750\n",
|
|
"367 252674.625\n",
|
|
"368 231582.750\n",
|
|
"369 210496.125\n",
|
|
"370 189430.500\n",
|
|
"371 168367.500\n",
|
|
"372 147359.625\n",
|
|
"373 126286.125\n",
|
|
"374 105196.875\n",
|
|
"375 84191.625\n",
|
|
"376 63157.500\n",
|
|
"377 42084.000\n",
|
|
"378 21068.250\n",
|
|
"379 0.000\n",
|
|
"Name: value, dtype: float64\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import pandas as pd\n",
|
|
"\n",
|
|
"# Re-loading the CSV files as the code execution state was reset\n",
|
|
"total_volume_path = '/Users/nico/Downloads/monthly_total.csv'\n",
|
|
"flow_rate_path = '/Users/nico/Downloads/flow_rate.csv'\n",
|
|
"\n",
|
|
"total_volume_df = pd.read_csv(total_volume_path)\n",
|
|
"flow_rate_df = pd.read_csv(flow_rate_path)\n",
|
|
"\n",
|
|
"# Convert timestamps to datetime\n",
|
|
"total_volume_df['timestamp'] = pd.to_datetime(total_volume_df['timestamp'], format=\"%m/%d/%Y %I:%M:%S %p\")\n",
|
|
"flow_rate_df['timestamp'] = pd.to_datetime(flow_rate_df['timestamp'], format=\"%m/%d/%Y %I:%M:%S %p\")\n",
|
|
"\n",
|
|
"# Filter out the erroneous data in total volume\n",
|
|
"total_volume_df = total_volume_df[total_volume_df['timestamp'] <= '2023-11-17 11:00:00']\n",
|
|
"\n",
|
|
"# Sort the flow rate data in ascending order by timestamp\n",
|
|
"flow_rate_df = flow_rate_df.sort_values(by='timestamp')\n",
|
|
"\n",
|
|
"# Calculate time differences and total volume for flow rate data\n",
|
|
"flow_rate_df['time_diff'] = flow_rate_df['timestamp'].diff().dt.total_seconds().div(60) # in minutes\n",
|
|
"flow_rate_df.iloc[0, flow_rate_df.columns.get_loc('time_diff')] = 0\n",
|
|
"flow_rate_df['total_volume'] = flow_rate_df['value'] * flow_rate_df['time_diff']\n",
|
|
"monthly_flow_rate_total = flow_rate_df.groupby(pd.Grouper(key='timestamp', freq='M'))['total_volume'].sum()\n",
|
|
"\n",
|
|
"# Adjusting the calculation for the total volume data\n",
|
|
"total_volume_df['reset'] = total_volume_df['value'].lt(total_volume_df['value'].shift())\n",
|
|
"total_volume_df['group'] = total_volume_df['reset'].cumsum()\n",
|
|
"last_values_before_reset = total_volume_df[total_volume_df['reset']].groupby('group').first()['value']\n",
|
|
"last_value_final_group = total_volume_df[~total_volume_df['reset']].iloc[-1]['value']\n",
|
|
"\"\"\"\n",
|
|
"last_values_before_reset = last_values_before_reset.append(pd.Series([last_value_final_group]))\n",
|
|
"monthly_total_corrected = last_values_before_reset.groupby(pd.Grouper(freq='M')).sum()\n",
|
|
"\n",
|
|
"# Compare with the flow rate total\n",
|
|
"comparison_df_corrected = pd.DataFrame({'Corrected Total Volume': monthly_total_corrected, 'Flow Rate Total': monthly_flow_rate_total})\n",
|
|
"comparison_df_corrected.reset_index() # Resetting index for better visualization\n",
|
|
"comparison_df_corrected.head()\n",
|
|
"\"\"\"\n",
|
|
"with pd.option_context('display.max_rows', None, 'display.max_columns', None): # more options can be specified also\n",
|
|
" print(last_values_before_reset)\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "tbDataCollector",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.10.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|