fetch_script

Fetching/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+requests = "*"
+pandas = "*"
+urllib3 = "*"
+
+[requires]
+python_version = "3.8"

Fetching/fetch_script.py

@@ -0,0 +1,196 @@
+# @file fetch_script.py  
+# @author: Marek Hamracek
+#
+#
+# SETTING UP CRON
+#
+#   crontab -e
+#       0 10 1 * * /path/to/python3 /path/to/fetch_script.py    #First day of any month at 10 AM
+#                                                               #Replace /path/to/
+#
+#
+#
+
+import requests
+import pandas as pd
+import urllib3
+import datetime
+import argparse
+
+# Disable the InsecureRequestWarning - http://wso.stanford.edu/Tilts.html site does not have a certificate
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+url_flow_speed = "https://omniweb.gsfc.nasa.gov/cgi/nx1.cgi"
+url_phi = "https://cosmicrays.oulu.fi/phi/Phi_Table_2017.txt"
+url_tilt_angle = "http://wso.stanford.edu/Tilts.html"
+
+# Define the payload for OMNIWEB form request
+payload = {
+    "activity": "retrieve",
+    "res": "daily",          
+    "spacecraft": "omni2_daily",  
+    "start_date": "20250101",  
+    "end_date": "20250110",    
+    "vars": "24", 
+}
+
+def connect_to_url(url, method='get', data=None, verify_ssl=True):
+    """Connect to a URL and handle exceptions."""
+    try:
+        if method == 'get':
+            response = requests.get(url, verify=verify_ssl)
+        elif method == 'post':
+            response = requests.post(url, data=data)
+        response.raise_for_status()
+        print(f"{url} connected successfully.")
+        return response
+    except requests.exceptions.RequestException as e:
+        print(f"Error connecting to {url}: {e}")
+        return None
+
+def parse_to_csv(html_content: str, table_start: str, table_end: str, header_start: str, output_file_path: str):
+    """Parse HTML content and save to CSV."""
+
+    try:
+        text_data = html_content.replace("\x00", "")
+        start_index, end_index = None, None
+        header = None
+        lines = text_data.splitlines()
+
+        for i, line in enumerate(lines):
+            if header_start in line:
+                header = line.split()
+
+            if table_end in line and start_index is not None:
+                end_index = i
+                break
+            elif table_start in line:
+                start_index = i
+
+        if start_index is None or end_index is None:
+            raise ValueError("Table boundaries not found.\n\n")            
+
+        if header is None:
+            print("Warning: No header found, generating default header.")
+
+        table_string = "\n".join(lines[start_index + 1:end_index])
+        rows = table_string.split("\n")
+        data = [row.split() for row in rows]
+
+        df = pd.DataFrame(data, columns=header)  
+        df.to_csv(output_file_path, index=False)
+        print("Table data were saved to:", output_file_path)
+
+        with open(args.log_file, "a") as log_file:
+            log_file.write("\nThe most up-to-date entry from " + output_file_path + ":\n" + str(df.tail(1)))
+
+        return df.tail(1)
+
+    except ValueError as e:
+        print(f"Value Error: {e}")
+    except Exception as e:
+        print(f"An unspecified error occurred: {e}")
+
+def compute_new_entry(last_entry_flow_speed, last_entry_phi, last_entry_tilt_angle):
+    """Compute and append a new entry to an existing CSV."""
+
+    filepath = args.csv_file
+
+    try:
+        df = pd.read_csv(filepath)
+
+        # Date format conversion
+        date = pd.to_datetime(last_entry_tilt_angle['Start'], format='%Y:%m:%d')
+
+        carrington_rotation = last_entry_tilt_angle.iloc[0]['Rot']
+        year_week = round(date.dt.year.iloc[0] + date.dt.day_of_year.iloc[0] / 365, 7)
+        modulation_potentional = last_entry_phi.iloc[0].dropna().iloc[-1] # Last non-NaN value #Geliosphere has some calcs that are yet to be considered
+        tilt_angle = last_entry_tilt_angle.iloc[0]['R_av']
+        year = date.dt.year.iloc[0]
+        month = date.dt.month.iloc[0]
+        day = date.dt.day.iloc[0]
+
+        # Polarity calculation based on a 11-year sun cycle
+        reference_year = 2012
+        years_since_ref = year - reference_year
+        if (years_since_ref // 11) % 2 == 0:
+            polarity = 1
+        else:
+            polarity = -1
+
+        if (args.E):
+            # 2D Solarprop-like Models
+            if polarity == -1:
+                k0_au2_s = (137/int(modulation_potentional)) - 0.061
+            elif polarity == 1:
+                k0_au2_s = 0.07 * (137/int(modulation_potentional)) - 0.061
+
+        else:
+            # 1D Models
+            k0_au2_s = (float(last_entry_flow_speed.iloc[0]['1']) * 6.68459e-9) * (99 / (3 * int(modulation_potentional))) 
+
+
+
+        new_row = pd.DataFrame({
+            'carrington_rotation': [carrington_rotation],
+            'year_week': [year_week],
+            'modulation_potentional': [modulation_potentional],
+            'k0_au2/s': [k0_au2_s],
+            'tilt_angle': [tilt_angle],
+            'year': [year],
+            'month': [month],
+            'day': [day],
+            'polarity': [polarity]
+        })
+
+        df = pd.concat([df, new_row], ignore_index=True)
+
+        print("\n Last entries: file: ",filepath," \n---------------------------------------------------")
+        print(df.tail(5),"\n")
+        df.to_csv(filepath, index=False)
+
+    except ValueError as e:
+        print(f"Value Error: {e}")
+    except Exception as e:
+        print(f"An error occurred when computing new entry: {e}")
+
+def log_start():
+    """Log the start of the script."""
+
+    with open(args.log_file, "w") as log_file:
+        log_file.write(f"\nScript started successfully at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
+
+def main():
+    """Main function to orchestrate the script."""
+
+    print("\n")
+    print(f"Script started successfully at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    print("\n")
+
+    log_start()
+
+    response_flow_speed = connect_to_url(url_flow_speed, method='post', data=payload)
+    response_phi = connect_to_url(url_phi, verify_ssl=False)
+    response_tilt_angle = connect_to_url(url_tilt_angle)
+
+    if response_flow_speed and response_phi and response_tilt_angle:
+        last_entry_flow_speed = parse_to_csv(response_flow_speed.text, "YEAR DOY", "</pre>", "YEAR DOY", "table_data/table_data_flow_speed.csv")
+        last_entry_phi = parse_to_csv(response_phi.text, "+++", "+++", "Year  Jan", "table_data/table_data_phi.csv")
+        last_entry_tilt_angle = parse_to_csv(response_tilt_angle.text, "Carr Rot", "</pre>", "Carr Rot", "table_data/table_data_tilt_angle.csv")
+        compute_new_entry(last_entry_flow_speed, last_entry_phi, last_entry_tilt_angle)
+
+        print(args.csv_file, "has been updated successfully")
+
+    print(args.log_file ," has been updated successfully\n")
+
+if __name__ == "__main__":
+    # Set up argument parser
+    parser = argparse.ArgumentParser(description="Fetch and process data.")
+    parser.add_argument('--csv_file', type=str, default='Geliosphere_K0_phi_table.csv', help="Path to the output CSV file")
+    parser.add_argument('--log_file', type=str, default='fetch_script_log.txt', help="Path to the output log file")
+
+    parser.add_argument('--E', action='store_true', help="Enable fetching data for 2D backward-in-time models")
+
+    args = parser.parse_args()
+
+    main()

Fetching/fetch_script_log.txt

@@ -0,0 +1,12 @@
+
+Script started successfully at 2025-04-27 19:08:45
+
+The most up-to-date entry from table_data/table_data_flow_speed.csv:
+   YEAR DOY HR     1
+9  2025  10  0  419.
+The most up-to-date entry from table_data/table_data_phi.csv:
+    Year  Jan  Feb  Mar  Apr  May  Jun  Jul  Aug  Sep  Oct  Nov  Dec Annual
+73  2024  764  770  766  759  812  766  728  788  849  906  894  881    807
+The most up-to-date entry from table_data/table_data_tilt_angle.csv:
+    Carr   Rot       Start Date  R_av   R_n    R_s  L_av   L_n    L_s
+651   CR  2293  2025:01:06  19h  61.2  53.1  -69.2  74.1  74.3  -74.0

Fetching/table_data/table_data_flow_speed.csv

@@ -0,0 +1,11 @@
+YEAR,DOY,HR,1
+2025,1,0,504.
+2025,2,0,509.
+2025,3,0,436.
+2025,4,0,536.
+2025,5,0,592.
+2025,6,0,541.
+2025,7,0,535.
+2025,8,0,450.
+2025,9,0,379.
+2025,10,0,419.

Fetching/table_data/table_data_phi.csv

@@ -0,0 +1,75 @@
+Year,Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec,Annual
+1951,N/A,661,651,642,601,571,600,623,580,574,588,573,606
+1952,599,609,622,595,557,540,528,529,521,558,541,553,563
+1953,575,563,569,565,564,553,561,556,553,543,545,533,557
+1954,527,515,497,504,499,501,495,480,485,487,495,504,499
+1955,539,505,503,503,496,500,500,508,496,516,513,549,511
+1956,580,606,656,613,637,627,602,607,632,580,663,775,632
+1957,856,863,840,914,872,902,933,893,1015,970,986,1046,924
+1958,1040,1021,1078,1061,978,937,1010,954,939,936,922,959,986
+1959,932,966,900,862,925,880,1071,1038,992,909,896,916,941
+1960,962,932,888,947,945,906,905,854,856,856,892,850,899
+1961,789,770,771,775,746,750,856,784,753,729,683,700,759
+1962,704,715,705,720,689,682,675,675,696,701,677,686,694
+1963,644,627,629,611,634,610,606,615,641,619,608,591,620
+1964,577,579,562,553,538,531,534,529,521,518,516,499,538
+1965,497,495,484,472,466,498,515,517,511,499,484,485,494
+1966,501,503,525,534,517,551,571,575,673,614,591,616,564
+1967,633,647,614,623,659,675,656,683,674,663,691,694,659
+1968,680,688,693,684,708,752,746,732,758,801,883,854,748
+1969,761,756,771,788,864,897,861,812,784,774,770,772,801
+1970,768,740,746,791,783,849,848,801,754,736,776,704,775
+1971,691,652,644,644,627,579,574,559,557,535,538,550,596
+1972,545,542,509,499,516,565,533,649,542,529,554,532,543
+1973,525,528,548,603,639,582,559,546,513,516,508,505,548
+1974,514,500,527,535,580,603,648,605,635,629,614,566,580
+1975,560,535,527,517,511,501,506,523,516,520,545,524,524
+1976,531,521,522,540,520,508,501,500,492,495,494,499,510
+1977,506,503,492,499,498,510,546,540,541,515,504,503,513
+1978,547,560,568,613,667,623,619,560,555,603,583,583,590
+1979,622,638,664,720,696,757,757,821,799,748,749,701,723
+1980,717,727,692,735,734,813,817,803,793,853,905,906,791
+1981,833,878,888,919,955,869,860,858,816,911,894,827,876
+1982,773,877,784,765,735,880,1012,1008,1082,1008,973,1029,911
+1983,936,876,816,807,890,834,776,777,748,743,733,731,806
+1984,710,722,763,791,859,807,784,747,731,730,739,725,759
+1985,709,665,654,637,629,596,606,606,575,571,554,561,614
+1986,564,627,578,524,515,511,508,507,503,488,524,491,528
+1987,466,446,447,451,470,507,527,551,571,571,597,594,517
+1988,665,640,627,642,636,646,691,703,694,712,723,796,681
+1989,828,832,980,956,1014,1000,912,952,986,1055,1126,1078,977
+1990,1018,1000,1034,1103,1123,1125,1036,1059,999,941,890,889,1018
+1991,826,814,1018,990,972,1360,1334,1133,990,953,943,895,1019
+1992,889,923,852,769,797,734,693,696,713,671,688,646,756
+1993,658,662,695,651,635,620,612,612,596,591,593,595,627
+1994,596,635,625,631,612,610,594,574,556,564,563,570,594
+1995,553,542,560,549,542,544,541,535,530,533,530,526,540
+1996,522,505,505,499,505,505,506,509,514,522,525,517,511
+1997,506,497,494,498,495,496,499,489,496,507,518,510,500
+1998,505,500,498,559,606,583,554,589,558,536,552,576,551
+1999,618,623,609,593,601,584,568,632,687,719,735,757,644
+2000,728,759,795,788,846,899,958,908,868,811,889,859,842
+2001,820,761,723,866,789,769,757,809,804,844,798,775,793
+2002,851,767,798,804,801,785,830,889,841,820,870,849,825
+2003,822,821,808,833,844,903,853,834,800,844,1026,840,852
+2004,852,766,715,691,664,663,700,685,660,601,667,651,693
+2005,757,675,656,634,692,647,663,686,755,629,603,603,667
+2006,582,552,522,519,506,508,521,519,514,496,500,546,524
+2007,492,498,484,463,459,456,459,463,453,455,462,453,466
+2008,462,465,462,464,465,463,457,445,437,437,429,426,451
+2009,421,414,407,398,389,393,393,396,389,390,397,390,398
+2010,398,436,448,475,462,471,478,489,483,483,496,499,468
+2011,488,486,513,568,534,599,576,568,568,590,549,516,546
+2012,542,574,644,557,555,590,662,649,610,610,601,587,598
+2013,571,565,595,590,667,675,656,644,630,598,610,635,620
+2014,630,670,650,639,620,658,640,607,640,636,648,703,645
+2015,674,672,709,684,654,664,628,622,623,612,603,590,645
+2016,554,526,531,529,524,525,537,519,518,497,482,483,519
+2017,465,458,451,466,460,453,477,505,522,493,466,460,473
+2018,459,451,448,442,446,431,433,433,429,425,430,434,439
+2019,434,438,421,421,444,426,426,429,419,414,418,417,426
+2020,410,398,402,391,399,408,411,412,413,416,416,431,409
+2021,435,464,430,419,422,428,438,428,432,432,453,443,436
+2022,449,472,474,482,514,551,579,607,583,563,566,622,539
+2023,634,686,694,662,676,680,712,749,758,720,760,775,709
+2024,764,770,766,759,812,766,728,788,849,906,894,881,807