{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "group\n",
      "1       126406.875\n",
      "2       105338.625\n",
      "3        84267.750\n",
      "4        63212.625\n",
      "5        42147.000\n",
      "6        21084.000\n",
      "7            0.000\n",
      "8       543558.750\n",
      "9       515849.250\n",
      "10      494773.125\n",
      "11      473647.125\n",
      "12      452282.250\n",
      "13      431227.125\n",
      "14      410216.625\n",
      "15      388972.500\n",
      "16      367639.125\n",
      "17      346610.250\n",
      "18      325187.625\n",
      "19      303883.125\n",
      "20      282849.000\n",
      "21      261846.375\n",
      "22      240363.375\n",
      "23      219006.375\n",
      "24      197497.125\n",
      "25      176457.750\n",
      "26      148561.875\n",
      "27      127115.625\n",
      "28      106081.500\n",
      "29       84666.750\n",
      "30       63490.875\n",
      "31       42406.875\n",
      "32       21404.250\n",
      "33           0.000\n",
      "34      487780.125\n",
      "35      466738.125\n",
      "36      445693.500\n",
      "37      422475.375\n",
      "38      401428.125\n",
      "39      380341.500\n",
      "40      359289.000\n",
      "41      338241.750\n",
      "42      317212.875\n",
      "43      296205.000\n",
      "44      275139.375\n",
      "45      254113.125\n",
      "46      248435.250\n",
      "47      227380.125\n",
      "48      206338.125\n",
      "49      185317.125\n",
      "50      164227.875\n",
      "51      143180.625\n",
      "52      122096.625\n",
      "53      101028.375\n",
      "54       84273.000\n",
      "55       63210.000\n",
      "56       42152.250\n",
      "57       21063.000\n",
      "58           0.000\n",
      "59      269338.125\n",
      "60      247128.000\n",
      "61      226057.125\n",
      "62      204975.750\n",
      "63      183962.625\n",
      "64      162873.375\n",
      "65      141862.875\n",
      "66      120849.750\n",
      "67       99789.375\n",
      "68       78705.375\n",
      "69       57647.625\n",
      "70       36639.750\n",
      "71       22120.875\n",
      "72           0.000\n",
      "73     6370203.000\n",
      "74     6349189.875\n",
      "75     6328179.375\n",
      "76     6307153.125\n",
      "77     6286082.250\n",
      "78     6273143.625\n",
      "79     6251915.250\n",
      "80     6230881.125\n",
      "81     6209854.875\n",
      "82     6188807.625\n",
      "83     6167731.500\n",
      "84     6146681.625\n",
      "85     6125613.375\n",
      "86     6104595.000\n",
      "87     6083526.750\n",
      "88     6062487.375\n",
      "89     6041437.500\n",
      "90     6020382.375\n",
      "91     5999327.250\n",
      "92     5978261.625\n",
      "93     5957224.875\n",
      "94     5936148.750\n",
      "95     5915075.250\n",
      "96     5894028.000\n",
      "97     5876582.250\n",
      "98     5855558.625\n",
      "99     5834516.625\n",
      "100    5813498.250\n",
      "101    5792445.750\n",
      "102    5771398.500\n",
      "103    5750351.250\n",
      "104    5729280.375\n",
      "105    5708217.375\n",
      "106    5687199.000\n",
      "107    5666128.125\n",
      "108    5645044.125\n",
      "109    5623970.625\n",
      "110    5602952.250\n",
      "111    5584931.625\n",
      "112    5563905.375\n",
      "113    5542900.125\n",
      "114    5521863.375\n",
      "115    5500829.250\n",
      "116    5479803.000\n",
      "117    5458782.000\n",
      "118    5437724.250\n",
      "119    5416650.750\n",
      "120    5395624.500\n",
      "121    5374606.125\n",
      "122    5353548.375\n",
      "123    5332482.750\n",
      "124    5311404.000\n",
      "125    5290401.375\n",
      "126    5269317.375\n",
      "127    5248233.375\n",
      "128    5227136.250\n",
      "129    5206128.375\n",
      "130    5185041.750\n",
      "131    5164028.625\n",
      "132    5143005.000\n",
      "133    5121944.625\n",
      "134    5100876.375\n",
      "135    5079855.375\n",
      "136    5058768.750\n",
      "137    5037755.625\n",
      "138    5016705.750\n",
      "139    4995655.875\n",
      "140    4974577.125\n",
      "141    4953535.125\n",
      "142    4932451.125\n",
      "143    4911361.875\n",
      "144    4890270.000\n",
      "145    4869212.250\n",
      "146    4859124.375\n",
      "147    4838116.500\n",
      "148    4817058.750\n",
      "149    4795974.750\n",
      "150    4774901.250\n",
      "151    4753851.375\n",
      "152    4732767.375\n",
      "153    4711722.750\n",
      "154    4690675.500\n",
      "155    4669581.000\n",
      "156    4648567.875\n",
      "157    4627546.875\n",
      "158    4606497.000\n",
      "159    4585439.250\n",
      "160    4564376.250\n",
      "161    4543334.250\n",
      "162    4522300.125\n",
      "163    4501279.125\n",
      "164    4480268.625\n",
      "165    4459205.625\n",
      "166    4438161.000\n",
      "167    4417087.500\n",
      "168    4396006.125\n",
      "169    4375003.500\n",
      "170    4353919.500\n",
      "171    4332861.750\n",
      "172    4311827.625\n",
      "173    4290748.875\n",
      "174    4269743.625\n",
      "175    4248672.750\n",
      "176    4227609.750\n",
      "177    4206583.500\n",
      "178    4185528.375\n",
      "179    4164470.625\n",
      "180    4143441.750\n",
      "181    4122436.500\n",
      "182    4102210.875\n",
      "183    4081184.625\n",
      "184    4060168.875\n",
      "185    4039134.750\n",
      "186    4018103.250\n",
      "187    3997090.125\n",
      "188    3976035.000\n",
      "189    3955000.875\n",
      "190    3933987.750\n",
      "191    3912953.625\n",
      "192    3891861.750\n",
      "193    3870817.125\n",
      "194    3849806.625\n",
      "195    3828790.875\n",
      "196    3807699.000\n",
      "197    3786617.625\n",
      "198    3765612.375\n",
      "199    3744583.500\n",
      "200    3723544.125\n",
      "201    3702454.875\n",
      "202    3681431.250\n",
      "203    3660420.750\n",
      "204    3639384.000\n",
      "205    3618352.500\n",
      "206    3598822.500\n",
      "207    3577777.875\n",
      "208    3556735.875\n",
      "209    3535722.750\n",
      "210    3514667.625\n",
      "211    3493659.750\n",
      "212    3472607.250\n",
      "213    3451575.750\n",
      "214    3435453.000\n",
      "215    3414371.625\n",
      "216    3393363.750\n",
      "217    3372355.875\n",
      "218    3351353.250\n",
      "219    3330292.875\n",
      "220    3309214.125\n",
      "221    3288211.500\n",
      "222    3267143.250\n",
      "223    3246082.875\n",
      "224    3225043.500\n",
      "225    3204001.500\n",
      "226    3182972.625\n",
      "227    3161928.000\n",
      "228    3140880.750\n",
      "229    3119830.875\n",
      "230    3098770.500\n",
      "231    3077702.250\n",
      "232    3056618.250\n",
      "233    3035547.375\n",
      "234    3014460.750\n",
      "235    2993387.250\n",
      "236    2972316.375\n",
      "237    2951308.500\n",
      "238    2930237.625\n",
      "239    2909221.875\n",
      "240    2888130.000\n",
      "241    2867038.125\n",
      "242    2845948.875\n",
      "243    2824938.375\n",
      "244    2803925.250\n",
      "245    2782846.500\n",
      "246    2761799.250\n",
      "247    2740757.250\n",
      "248    2718371.250\n",
      "249    2697321.375\n",
      "250    2676255.750\n",
      "251    2655213.750\n",
      "252    2634129.750\n",
      "253    2613082.500\n",
      "254    2592051.000\n",
      "255    2570990.625\n",
      "256    2549946.000\n",
      "257    2528911.875\n",
      "258    2507904.000\n",
      "259    2486848.875\n",
      "260    2465801.625\n",
      "261    2444770.125\n",
      "262    2423678.250\n",
      "263    2402675.625\n",
      "264    2381646.750\n",
      "265    2360596.875\n",
      "266    2339568.000\n",
      "267    2318484.000\n",
      "268    2297463.000\n",
      "269    2276457.750\n",
      "270    2255407.875\n",
      "271    2234400.000\n",
      "272    2213397.375\n",
      "273    2192350.125\n",
      "274    2171255.625\n",
      "275    2150216.250\n",
      "276    2129142.750\n",
      "277    2108082.375\n",
      "278    2087011.500\n",
      "279    2065966.875\n",
      "280    2044859.250\n",
      "281    2023798.875\n",
      "282    2002712.250\n",
      "283    1981665.000\n",
      "284    1963833.375\n",
      "285    1942799.250\n",
      "286    1921736.250\n",
      "287    1900660.125\n",
      "288    1879594.500\n",
      "289    1858523.625\n",
      "290    1837479.000\n",
      "291    1816450.125\n",
      "292    1795360.875\n",
      "293    1783123.125\n",
      "294    1782676.875\n",
      "295    1761671.625\n",
      "296    1740627.000\n",
      "297    1719606.000\n",
      "298    1698585.000\n",
      "299    1677548.250\n",
      "300    1656519.375\n",
      "301    1635477.375\n",
      "302    1614451.125\n",
      "303    1593359.250\n",
      "304    1572325.125\n",
      "305    1551285.750\n",
      "306    1530228.000\n",
      "307    1509146.625\n",
      "308    1488109.875\n",
      "309    1467091.500\n",
      "310    1446041.625\n",
      "311    1424999.625\n",
      "312    1403976.000\n",
      "313    1383823.875\n",
      "314    1362737.250\n",
      "315    1341648.000\n",
      "316    1320553.500\n",
      "317    1299498.375\n",
      "318    1278482.625\n",
      "319    1257414.375\n",
      "320    1236343.500\n",
      "321    1215272.625\n",
      "322    1194217.500\n",
      "323    1173133.500\n",
      "324    1152054.750\n",
      "325    1130997.000\n",
      "326    1109913.000\n",
      "327    1088844.750\n",
      "328    1067755.500\n",
      "329    1046729.250\n",
      "330    1025640.000\n",
      "331    1004556.000\n",
      "332     983487.750\n",
      "333     962456.250\n",
      "334     941445.750\n",
      "335     920361.750\n",
      "336     899277.750\n",
      "337     878175.375\n",
      "338     857128.125\n",
      "339     836122.875\n",
      "340     815091.375\n",
      "341     794044.125\n",
      "342     773036.250\n",
      "343     752007.375\n",
      "344     731004.750\n",
      "345     709957.500\n",
      "346     688949.625\n",
      "347     673727.250\n",
      "348     652666.875\n",
      "349     631632.750\n",
      "350     610598.625\n",
      "351     589509.375\n",
      "352     568480.500\n",
      "353     547407.000\n",
      "354     526375.500\n",
      "355     505336.125\n",
      "356     484273.125\n",
      "357     463199.625\n",
      "358     442139.250\n",
      "359     421086.750\n",
      "360     400008.000\n",
      "361     378973.875\n",
      "362     357895.125\n",
      "363     336882.000\n",
      "364     315819.000\n",
      "365     294771.750\n",
      "366     273708.750\n",
      "367     252674.625\n",
      "368     231582.750\n",
      "369     210496.125\n",
      "370     189430.500\n",
      "371     168367.500\n",
      "372     147359.625\n",
      "373     126286.125\n",
      "374     105196.875\n",
      "375      84191.625\n",
      "376      63157.500\n",
      "377      42084.000\n",
      "378      21068.250\n",
      "379          0.000\n",
      "Name: value, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Re-loading the CSV files as the code execution state was reset\n",
    "total_volume_path = '/Users/nico/Downloads/monthly_total.csv'\n",
    "flow_rate_path = '/Users/nico/Downloads/flow_rate.csv'\n",
    "\n",
    "total_volume_df = pd.read_csv(total_volume_path)\n",
    "flow_rate_df = pd.read_csv(flow_rate_path)\n",
    "\n",
    "# Convert timestamps to datetime\n",
    "total_volume_df['timestamp'] = pd.to_datetime(total_volume_df['timestamp'], format=\"%m/%d/%Y %I:%M:%S %p\")\n",
    "flow_rate_df['timestamp'] = pd.to_datetime(flow_rate_df['timestamp'], format=\"%m/%d/%Y %I:%M:%S %p\")\n",
    "\n",
    "# Filter out the erroneous data in total volume\n",
    "total_volume_df = total_volume_df[total_volume_df['timestamp'] <= '2023-11-17 11:00:00']\n",
    "\n",
    "# Sort the flow rate data in ascending order by timestamp\n",
    "flow_rate_df = flow_rate_df.sort_values(by='timestamp')\n",
    "\n",
    "# Calculate time differences and total volume for flow rate data\n",
    "flow_rate_df['time_diff'] = flow_rate_df['timestamp'].diff().dt.total_seconds().div(60)  # in minutes\n",
    "flow_rate_df.iloc[0, flow_rate_df.columns.get_loc('time_diff')] = 0\n",
    "flow_rate_df['total_volume'] = flow_rate_df['value'] * flow_rate_df['time_diff']\n",
    "monthly_flow_rate_total = flow_rate_df.groupby(pd.Grouper(key='timestamp', freq='M'))['total_volume'].sum()\n",
    "\n",
    "# Adjusting the calculation for the total volume data\n",
    "total_volume_df['reset'] = total_volume_df['value'].lt(total_volume_df['value'].shift())\n",
    "total_volume_df['group'] = total_volume_df['reset'].cumsum()\n",
    "last_values_before_reset = total_volume_df[total_volume_df['reset']].groupby('group').first()['value']\n",
    "last_value_final_group = total_volume_df[~total_volume_df['reset']].iloc[-1]['value']\n",
    "\"\"\"\n",
    "last_values_before_reset = last_values_before_reset.append(pd.Series([last_value_final_group]))\n",
    "monthly_total_corrected = last_values_before_reset.groupby(pd.Grouper(freq='M')).sum()\n",
    "\n",
    "# Compare with the flow rate total\n",
    "comparison_df_corrected = pd.DataFrame({'Corrected Total Volume': monthly_total_corrected, 'Flow Rate Total': monthly_flow_rate_total})\n",
    "comparison_df_corrected.reset_index()  # Resetting index for better visualization\n",
    "comparison_df_corrected.head()\n",
    "\"\"\"\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "    print(last_values_before_reset)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tbDataCollector",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}