Commit Live PR

data/scraped.csv

@@ -1,4 +1,4 @@
-,United States of America,Federal state,US USA 840,US,00,,,U.S.,U.S.,U.S.A.,,,,,
+,United States of America,Federal state,".mw-parser-output .monospaced{font-family:monospace,monospace}USUSA840",US,00,,,U.S.,U.S.,U.S.A.,,,,,
 12,Alabama,State,US-AL,AL,01,AL,AL,Ala.,Ala.,,,,,,
 13,Alaska,State,US-AK,AK,02,AK,AK,Alaska,Alaska,Alas.,,,,,
 14,Arizona,State,US-AZ,AZ,04,AZ,AZ,Ariz.,Ariz.,Az.,,,,,
@@ -9,7 +9,7 @@
 19,Delaware,State,US-DE,DE,10,DE,DL,Del.,Del.,De.,,,,,
 20,DistrictofColumbia,Federal district,US-DC,DC,11,DC,DC,D.C.,D.C.,Wash. D.C.,,,,,
 21,Florida,State,US-FL,FL,12,FL,FL,Fla.,Fla.,"Fl., Flor.",,,,,
-22,Georgia,State,US-GA,GA,13,GA,GA,Ga.,Ga.,,,,,,
+22,Georgia,State,US-GA,GA,13,GA,GA,Ga.,Ga.,Geo.,,,,,
 23,Hawaii,State,US-HI,HI,15,HI,HA,Hawaii,Hawaii,H.I.,,,,,
 24,Idaho,State,US-ID,ID,16,ID,ID,Idaho,Idaho,"Id., Ida.",,,,,
 25,Illinois,State,US-IL,IL,17,IL,IL,Ill.,Ill.,"Il., Ills., Ill's",,,,,
@@ -50,12 +50,12 @@
 60,WestVirginia,State,US-WV,WV,54,WV,WV,W. Va.,W.Va.,"W.V., W. Virg.",,,,,
 61,Wisconsin,State,US-WI,WI,55,WI,WS,Wis.,Wis.,"Wi., Wisc.",,,,,
 62,Wyoming,State,US-WY,WY,56,WY,WY,Wyo.,Wyo.,Wy.,,,,,
-63,AmericanSamoa,Insular area (Territory),AS ASM 016 US-AS,AS,60,AS,AS,A.S.,,,,,,,
-64,Guam,Insular area (Territory),GU GUM 316 US-GU,GU,66,GU,GU,Guam,,,,,,,
-65,NorthernMarianaIslands,Insular area (Commonwealth),MP MNP 580 US-MP,MP,69,MP,CM,M.P.,,CNMI[3],,,,,
-66,PuertoRico,Insular area (Territory),PR PRI 630 US-PR,PR,72,PR,PR,P.R.,,,,,,,
-67,U.S.VirginIslands,Insular area (Territory),VI VIR 850 US-VI,VI,78,VI,VI,V.I.,,U.S.V.I.,,,,,
-68,U.S.MinorOutlyingIslands,Insular areas,UM UMI 581 US-UM,UM,74,,,,,,,,,,
+63,AmericanSamoa,Insular area (Territory),ASASM016US-AS,AS,60,AS,AS,A.S.,,,,,,,
+64,Guam,Insular area (Territory),GUGUM316US-GU,GU,66,GU,GU,Guam,,,,,,,
+65,NorthernMarianaIslands,Insular area (Commonwealth),MPMNP580US-MP,MP,69,MP,CM,M.P.,,CNMI[3],,,,,
+66,PuertoRico,Insular area (Territory),PRPRI630US-PR,PR,72,PR,PR,P.R.,,,,,,,
+67,U.S.VirginIslands,Insular area (Territory),VIVIR850US-VI,VI,78,VI,VI,V.I.,,U.S.V.I.,,,,,
+68,U.S.MinorOutlyingIslands,Insular areas,UMUMI581US-UM,UM,74,,,,,,,,,,
 69,BakerIsland,island,UM-81,,81,,,,,XB[4],,,,,
 70,HowlandIsland,island,UM-84,,84,,,,,XH[4],,,,,
 71,JarvisIsland,island,UM-86,,86,,,,,XQ[4],,,,,
@@ -65,14 +65,14 @@
 75,NavassaIsland,island,UM-76,,76,,,,,XV[4],,,,,
 76,PalmyraAtoll[5],atoll[5],UM-95,,95,,,,,XL[4],,,,,
 77,WakeIsland,atoll,UM-79,,79,,,,,QW[4],,,,,
-78,Micronesia,Freely associated state,FM FSM 583,FM,64,FM,,,,,,,,,
-79,MarshallIslands,Freely associated state,MH MHL 584,MH,68,MH,,,,,,,,,
-80,Palau,Freely associated state,PW PLW 585,PW,70,PW,,,,,,,,,
+78,Micronesia,Freely associated state,FMFSM583,FM,64,FM,,,,,,,,,
+79,MarshallIslands,Freely associated state,MHMHL584,MH,68,MH,,,,,,,,,
+80,Palau,Freely associated state,PWPLW585,PW,70,PW,,,,,,,,,
 81,U.S.ArmedForces–Americas[6],US military mail code,,,,AA,,,,,,,,,
 82,U.S.ArmedForces–Europe[7],US military mail code,,,,AE,,,,,,,,,
 83,U.S.ArmedForces–Pacific[8],US military mail code,,,,AP,,,,,,,,,
 84,NorthernMarianaIslands,Obsolete postal code[9],,,,CM,,,,,,,,,
-85,PanamaCanalZone,Obsolete postal code,PZ PCZ 594,,,CZ,,,,,,,,,
+85,PanamaCanalZone,Obsolete postal code,PZPCZ594,,,CZ,,,,,,,,,
 86,Nebraska,Obsolete postal code[10],,,,NB,,,,,,,,,
-87,PhilippineIslands,Obsolete postal code,PH PHL 608[11],,,PI,,,,,,,,,
-88,TrustTerritoryofthePacificIslands,Obsolete postal code,PC PCI 582,,,TT,,,,,,,,,
+87,PhilippineIslands,Obsolete postal code,PHPHL608[11],,,PI,,,,,,,,,
+88,TrustTerritoryofthePacificIslands,Obsolete postal code,PCPCI582,,,TT,,,,,,,,,

q01_load_data/build.py

@@ -1,5 +1,18 @@
+# %load q01_load_data/build.py
 import pandas as pd
+import numpy as np
+
+def q01_load_data(path='data/excel-comp-data.xlsx'):
+    #'write your solution here'
+    df=pd.read_excel(path)
+    df=pd.DataFrame(df)
+    df['state']=df['state'].str.lower()#Series column lower case syntax is str.lower()
+    df['Total']=df['Jan']+df['Feb']+df['Mar']#+ operator works coz its overloaded
+    return df
+
+c=q01_load_data(path='data/excel-comp-data.xlsx')
+c.shape
+
+
 
 
-def q01_load_data(path):
-    "write your solution here"

q02_append_row/build.py

@@ -1,11 +1,23 @@
+# %load q02_append_row/build.py
 import pandas as pd
 import sys, os
 #sys.path.append(os.path.join(os.path.dirname(os.curdir)))
 from greyatomlib.pandas_guided_project.q01_load_data.build import q01_load_data
+df= q01_load_data(path='data/excel-comp-data.xlsx')
 
+def q02_append_row(path='data/excel-comp-data.xlsx'):
+    a=df.append(df[['Jan','Feb','Mar','total']].sum(),ignore_index=True)
+    return a
+#     a=df.append([df[['Jan','Feb','Mar']].sum()],ignore_index=True)<--This is how you create a new row in a dataframe
+
+
 
-def q02_append_row(path):
-    "write your solution here"
+
+
+
+
+c=q02_append_row(path='data/excel-comp-data.xlsx')
+c
 
 
 

q03_scrape_clean/build.py

@@ -1,9 +1,42 @@
+# %load q03_scrape_clean/build.py
 import pandas as pd
 import sys, os
 import requests
 sys.path.append(os.path.join(os.path.dirname(os.curdir)))
+from bs4 import BeautifulSoup
+
+def q03_scrape_clean(url='https://en.wikipedia.org/wiki/List_of_U.S._state_abbreviations'):
+    a=requests.get(url)
+    soup=BeautifulSoup(a.content,'html.parser')
+    stat=soup.find_all('table',class_='wikitable')[0]#.tbody.tr.decompose()
+    #stat.tbody.tr.decompose()
+    df=pd.read_html(str(stat))[0]
+    df=pd.DataFrame(df)
+    df=df[9:]
+    df=df.drop([10,9,11])#<--Directly pass a number and not a list for a single row
+    df=df.reset_index(drop=True)
+    df[0].str.replace(' ','')
+    return df
+
+
+#     rows=stat[0].tbody.find_all('tr')
+#     headers=stat[0].tbody.find_all('td')
+#     for row in rows:
+#         for cell in headers:
+
+#     res=requests.get(url)
+#     soup=BeautifulSoup(res.content,'lxml')
+#     table=soup.find_all('table')[0]
+#     return table
+
+
+
+
+
+
+
+c=q03_scrape_clean(url='https://en.wikipedia.org/wiki/List_of_U.S._state_abbreviations')
+c
 
 
-def q03_scrape_clean(url):
-    "write your solution here"
 

q04_mapping/build.py

@@ -1,10 +1,71 @@
+# %load q04_mapping/build.py
 import pandas as pd
 import sys, os
 import numpy as np
 #sys.path.append(os.path.join(os.path.dirname(os.curdir)))
+from greyatomlib.pandas_guided_project.q01_load_data.build import q01_load_data
 from greyatomlib.pandas_guided_project.q02_append_row.build import q02_append_row
-def q04_mapping(path1,path2):
-    "write your solution here"
+from greyatomlib.pandas_guided_project.q03_scrape_clean.build import q03_scrape_clean
+df= q01_load_data(path='data/excel-comp-data.xlsx')
+a=df.append(df[['Jan','Feb','Mar','total']].sum(),ignore_index=True)
+a
+b=q03_scrape_clean(url='https://en.wikipedia.org/wiki/List_of_U.S._state_abbreviations' )
+b.reset_index(drop=True)
+def q04_mapping(path1='data/excel-comp-data.xlsx',path2='data/scraped.csv'):
+#         mapping={}
+#         for i,j in zip(b['United States of America'],b['US']):
+#             mapping[i]=j
+
+#         state=[]
+#         for i in (a['state']):
+#             state.append(i)
+
+#         f=[]
+#         # for i,j in mapping.items():
+#         #     for k in state:
+#         #         print(k)
+#         del state[15]
+#         for i in state:
+#             f.append(i.capitalize())
+
+#         for i,j in enumerate(f):
+#             if j=='Northcarolina':
+#                 f[i]='NorthCarolina'
+#             elif j=='Mississipi':
+#                 f[i]='Mississippi'
+#             elif j=='Rhodeisland':
+#                 f[i]='RhodeIsland'
+#             elif j=='Tenessee':
+#                 f[i]='Tennessee'
+#             elif j=='Northdakota':
+#                 f[i]='NorthDakota'
+#         g=[]
+#         for j in f:
+#             for i in mapping.items():
+#                 if j==i[0]:
+#                     g.append(i[1])
+#         g.append('NaN')
+#         a.insert(loc=6,column='abbr',value=g)
+#         return a
+    df1=pd.read_excel(path1)
+    df1['total']=df1['Jan']+df1['Feb']+df1['Mar']
+    df1.loc[len(df1),:]=df1.sum()
+    df2=pd.read_csv(path2)
+    abbr_dict=dict(zip(df2.iloc[:,[1,6]]['United States of America'],df2['US']))
+    dftemp=df1['state'].map(abbr_dict)
+    df1.insert(loc=6,column='abbr',value=dftemp)
+    return df1
+
+
+
+
+
+
+c=q04_mapping(path1='data/excel-comp-data.xlsx',path2='data/scraped.csv')
+c
+
+
+
 
 
 

q05_replace_missing_values/build.py

@@ -1,3 +1,4 @@
+# %load q05_replace_missing_values/build.py
 import pandas as pd
 import numpy as np
 import sys
@@ -8,6 +9,67 @@
 path1 = 'data/excel-comp-data.xlsx'
 path2 = 'data/scraped.csv'
 def q05_replace_missing_values(path1,path2):
+        df1=pd.read_excel(path1)
+        df1['total']=df1['Jan']+df1['Feb']+df1['Mar']
+        df1.loc[len(df1),:]=df1.sum()
+        df2=pd.read_csv(path2)
+        abbr_dict=dict(zip(df2.iloc[:,[1,6]]['United States of America'],df2['US']))#
+        f=list(df1['state'])
+        for i,j in enumerate(f):
+            if j=='Northcarolina':
+                f[i]='NorthCarolina'
+            elif j=='Mississipi':
+                f[i]='Mississippi'
+            elif j=='Rhodeisland':
+                f[i]='RhodeIsland'
+            elif j=='Tenessee':
+                f[i]='Tennessee'
+            elif j=='Northdakota':
+                f[i]='NorthDakota'
+        df1['state']=f
+        dftemp=df1['state'].map(abbr_dict)
+        df1.insert(loc=6,column='abbr',value=dftemp)
+        return df1
+
+#         mapping={}
+#         for i,j in zip(df2['United States of America'],df2['US']):
+#             mapping[i]=j
+
+#         state=[]
+#         for i in (df1['state']):
+#             state.append(i)
+
+#         f=[]
+#         # for i,j in mapping.items():
+#         #     for k in state:
+#         #         print(k)
+# #         del state[15]
+#         for i in state:
+#             f.append(i.capitalize())
+
+#         for i,j in enumerate(f):
+#             if j=='Northcarolina':
+#                 f[i]='NorthCarolina'
+#             elif j=='Mississipi':
+#                 f[i]='Mississippi'
+#             elif j=='Rhodeisland':
+#                 f[i]='RhodeIsland'
+#             elif j=='Tenessee':
+#                 f[i]='Tennessee'
+#             elif j=='Northdakota':
+#                 f[i]='NorthDakota'
+#         g=[]
+#         for j in f:
+#             for i in mapping.items():
+#                 if j==i[0]:
+#                     g.append(i[1])
+#         g.append('NaN')
+#         df1.insert(loc=6,column='abbr',value=g)
+#         return df1
+
+
+
+c=q05_replace_missing_values(path1,path2)
+c
 
 
-#print(q05_replace_missing_values(path1,path2).shape)

q06_sub_total/build.py

@@ -1,3 +1,4 @@
+# %load q06_sub_total/build.py
 import pandas as pd
 import numpy as np
 from sklearn.model_selection import train_test_split
@@ -10,7 +11,32 @@
 path2 = 'data/scraped.csv'
 
 def q06_sub_total(path1,path2):
-    "write your solution here"
+        df1=pd.read_excel(path1)
+        df1['total']=df1['Jan']+df1['Feb']+df1['Mar']
+        df1.loc[len(df1),:]=df1.sum()
+        df2=pd.read_csv(path2)
+        abbr_dict=dict(zip(df2.iloc[:,[1,6]]['United States of America'],df2['US']))#
+        f=list(df1['state'])
+        for i,j in enumerate(f):
+            if j=='Northcarolina':
+                f[i]='NorthCarolina'
+            elif j=='Mississipi':
+                f[i]='Mississippi'
+            elif j=='Rhodeisland':
+                f[i]='RhodeIsland'
+            elif j=='Tenessee':
+                f[i]='Tennessee'
+            elif j=='Northdakota':
+                f[i]='NorthDakota'
+        df1['state']=f
+        dftemp=df1['state'].map(abbr_dict)
+        df1.insert(loc=6,column='abbr',value=dftemp)
+        groupby_abbr_sum=df1.groupby('abbr')[['Jan','Feb','Mar','total']].sum()
+        return groupby_abbr_sum
 
 
 
+c=q06_sub_total(path1,path2)
+c
+
+

q07_symbols/build.py

@@ -1,18 +1,53 @@
+# %load q07_symbols/build.py
 import pandas as pd
 import numpy as np
 from sklearn.model_selection import train_test_split
 import sys
 import os
-sys.path.append(os.path.join(os.path.dirname(os.curdir)))
-from greyatomlib.pandas_guided_project.q06_sub_total.build import q06_sub_total
+# sys.path.append(os.path.join(os.path.dirname(os.curdir)))
+# from greyatomlib.pandas_guided_project.q06_sub_total.build import q06_sub_total
 
 path1 = 'data/excel-comp-data.xlsx'
 path2 = 'data/scraped.csv'
 
 def q07_symbols(path1,path2):
-    "write your solution here"
+    df1=pd.read_excel(path1)
+    df1['total']=df1['Jan']+df1['Feb']+df1['Mar']
+    df1.loc[len(df1),:]=df1.sum()
+    df2=pd.read_csv(path2)
+    abbr_dict=dict(zip(df2.iloc[:,[1,6]]['United States of America'],df2['US']))#
+    f=list(df1['state'])
+    for i,j in enumerate(f):
+        if j=='Northcarolina':
+            f[i]='NorthCarolina'
+        elif j=='Mississipi':
+            f[i]='Mississippi'
+        elif j=='Rhodeisland':
+            f[i]='RhodeIsland'
+        elif j=='Tenessee':
+            f[i]='Tennessee'
+        elif j=='Northdakota':
+            f[i]='NorthDakota'
+    df1['state']=f
+    dftemp=df1['state'].map(abbr_dict)
+    df1.insert(loc=6,column='abbr',value=dftemp)
+    groupby_abbr_sum=df1.groupby('abbr')[['Jan','Feb','Mar','total']].sum()
+    df1[['Jan','Feb','Mar','total']]=df1[['Jan','Feb','Mar','total']].applymap(lambda x:('$'+str(int(x))))
+    f=list(df1['total'])
+    for i,j in enumerate(f):
+        if j=='05000':
+            f[i]='05,000'
+    df1['total']=pd.DataFrame(f)
+    return df1
+
+
 
+
 
 
-#print(q07_symbols(path1,path2))
+
+c=q07_symbols(path1,path2)
+
+c
+