Feat/extract website developer company

README.md

@@ -15,7 +15,10 @@ From the project root:
 
 ```bash
 python3 -m venv .venv
-source .venv/bin/activate   # Windows: .venv\Scripts\activate
+# Unix:    
+source .venv/bin/activate  
+# Windows: 
+.venv\Scripts\activate
 pip install -e .
 ```
 

data/mins_depts_test_check.csv

@@ -0,0 +1,50 @@
+Type,Institution Name,URL,Scrape_URL
+Ministry,Ministry of Defence,defence.lk,
+Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,
+Ministry,Ministry of Digital Economy,midec.gov.lk,
+Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,https://www.mfa.gov.lk/en
+Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,https://moe.gov.lk/en/
+Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,
+Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,https://transport.gov.lk/web/index.php?lang=en
+Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,https://pubad.gov.lk/web/index.php?lang=en
+Ministry,Ministry of Health and Mass Media,health.gov.lk,https://www.health.gov.lk/home/
+Ministry,Ministry of Justice and National Integration,moj.gov.lk,
+Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,
+Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,https://www.industry.gov.lk/web/
+Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,
+Ministry,Ministry of Environment,env.gov.lk,https://env.gov.lk/web/index.php?lang=en
+Ministry,Ministry of Energy,energy.gov.lk,
+Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,
+Ministry,Ministry of Labor,labourmin.gov.lk,
+Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,
+Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,
+Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,
+Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,
+Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,https://socialservices.gov.lk/web/index.php?lang=en
+Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,
+Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,
+Ministry,Ministry of Science and Technology,most.gov.lk,https://most.gov.lk/web/index.php?lang=en
+Department,Department of Immigration and Emigration,immigration.gov.lk,https://immigration.gov.lk/index_e.php
+Department,Department of Motor Traffic,dmt.gov.lk,
+Department,Department of Registration of Persons,drp.gov.lk,https://drp.gov.lk/en/home.php
+Department,Department of Registrar General,rgd.gov.lk,https://rgd.gov.lk/web/index.php?lang=en
+Department,Department of Inland Revenue,ird.gov.lk,
+Department,Sri Lanka Customs,customs.gov.lk,
+Department,Department of Pensions,pensions.gov.lk,
+Department,Department of Census and Statistics,statistics.gov.lk,
+Department,Department of Government Information,dgi.gov.lk,
+Department,Department of Government Printing,documents.gov.lk,
+Department,Department of Excise,excise.gov.lk,
+Department,Department of National Budget,nationalbudget.gov.lk,
+Department,Department of Meteorology,meteo.gov.lk,
+Department,Department of Wildlife Conservation,dwc.gov.lk,
+Department,Department of Forest Conservation,forestdept.gov.lk,
+Department,Department of National Zoological Gardens,colombozoo.gov.lk,
+Department,Department of Examinations,doenets.lk,
+Department,Department of Archaeology,archaeology.gov.lk,
+Department,Department of Samurdhi,samurdhi.gov.lk,https://samurdhi.gov.lk/web/index.php?lang=en
+Department,LSF,opensource.lk,
+Expired,Expired SSL,expired.badssl.com,
+Self-Signed,Self-Signed SSL,self-signed.badssl.com,
+Wrong Host,Wrong Host SSL,wrong.host.badssl.com,
+Never SSL,Never SSL,neverssl.com,

data/mins_depts_test_check_scored.csv

@@ -0,0 +1,50 @@
+Type,Institution Name,URL,Scrape_URL,footer_vendor,footer_vendor_error
+Ministry,Ministry of Defence,defence.lk,,unknown,
+Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,,found,elysiancrest.com
+Ministry,Ministry of Digital Economy,midec.gov.lk,,unreachable,Failed to fetch page
+Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,https://www.mfa.gov.lk/en,found,Frontwalker
+Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,https://moe.gov.lk/en/,unknown,
+Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,,found,onzdev.com
+Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,https://transport.gov.lk/web/index.php?lang=en,found,Procons Infotech
+Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,https://pubad.gov.lk/web/index.php?lang=en,found,gnu.org
+Ministry,Ministry of Health and Mass Media,health.gov.lk,https://www.health.gov.lk/home/,found,Weblankan & Arrogance Technologies (Pvt)Ltd
+Ministry,Ministry of Justice and National Integration,moj.gov.lk,,found,Procons Infotech
+Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,,unknown,
+Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,https://www.industry.gov.lk/web/,found,theekshana.lk
+Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,,unreachable,Failed to fetch page
+Ministry,Ministry of Environment,env.gov.lk,https://env.gov.lk/web/index.php?lang=en,found,Procons Infotech.
+Ministry,Ministry of Energy,energy.gov.lk,,unknown,
+Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,,unreachable,Failed to fetch page
+Ministry,Ministry of Labor,labourmin.gov.lk,,unknown,
+Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,,found,University of Kelaniya
+Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,,unreachable,Failed to fetch page
+Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,,self_built,IT UNIT Last Update
+Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,,unreachable,Failed to fetch page
+Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,https://socialservices.gov.lk/web/index.php?lang=en,found,Procons Infotech
+Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,,unreachable,Failed to fetch page
+Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,,unreachable,Failed to fetch page
+Ministry,Ministry of Science and Technology,most.gov.lk,https://most.gov.lk/web/index.php?lang=en,found,Procons Infotech
+Department,Department of Immigration and Emigration,immigration.gov.lk,https://immigration.gov.lk/index_e.php,found,SLT
+Department,Department of Motor Traffic,dmt.gov.lk,,found,Procons Infotech
+Department,Department of Registration of Persons,drp.gov.lk,https://drp.gov.lk/en/home.php,unknown,
+Department,Department of Registrar General,rgd.gov.lk,https://rgd.gov.lk/web/index.php?lang=en,unknown,
+Department,Department of Inland Revenue,ird.gov.lk,,unreachable,Failed to fetch page
+Department,Sri Lanka Customs,customs.gov.lk,,unknown,
+Department,Department of Pensions,pensions.gov.lk,,found,Procons Infotech
+Department,Department of Census and Statistics,statistics.gov.lk,,unreachable,Failed to fetch page
+Department,Department of Government Information,dgi.gov.lk,,unknown,
+Department,Department of Government Printing,documents.gov.lk,,unknown,
+Department,Department of Excise,excise.gov.lk,,found,LankaCom
+Department,Department of National Budget,nationalbudget.gov.lk,,unreachable,Failed to fetch page
+Department,Department of Meteorology,meteo.gov.lk,,unknown,
+Department,Department of Wildlife Conservation,dwc.gov.lk,,unreachable,Failed to fetch page
+Department,Department of Forest Conservation,forestdept.gov.lk,,found,ICTA & Procons Infotech
+Department,Department of National Zoological Gardens,colombozoo.gov.lk,,unreachable,Failed to fetch page
+Department,Department of Examinations,doenets.lk,,unknown,
+Department,Department of Archaeology,archaeology.gov.lk,,found,University of Kelaniya
+Department,Department of Samurdhi,samurdhi.gov.lk,https://samurdhi.gov.lk/web/index.php?lang=en,found,Information and Communication Technology Agency
+Department,LSF,opensource.lk,,unknown,
+Expired,Expired SSL,expired.badssl.com,,unknown,
+Self-Signed,Self-Signed SSL,self-signed.badssl.com,,unknown,
+Wrong Host,Wrong Host SSL,wrong.host.badssl.com,,unknown,
+Never SSL,Never SSL,neverssl.com,,unknown,

data/mins_depts_test_scored.csv

@@ -1,50 +1,50 @@
-Type,Institution Name,URL,ssl_status,ssl_error
-Ministry,Ministry of Defence,defence.lk,valid,
-Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid,
-Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid,
-Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid,
-Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid,
-Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid,
-Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid,
-Ministry,Ministry of Health and Mass Media,health.gov.lk,valid,
-Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid,
-Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid,
-Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid,
-Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,Ministry of Environment,env.gov.lk,valid,
-Ministry,Ministry of Energy,energy.gov.lk,valid,
-Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,Ministry of Labor,labourmin.gov.lk,valid,
-Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'."
-Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid,
-Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid,
-Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Ministry,Ministry of Science and Technology,most.gov.lk,valid,
-Department,Department of Immigration and Emigration,immigration.gov.lk,valid,
-Department,Department of Motor Traffic,dmt.gov.lk,valid,
-Department,Department of Registration of Persons,drp.gov.lk,valid,
-Department,Department of Registrar General,rgd.gov.lk,valid,
-Department,Department of Inland Revenue,ird.gov.lk,valid,
-Department,Sri Lanka Customs,customs.gov.lk,valid,
-Department,Department of Pensions,pensions.gov.lk,valid,
-Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[Errno 54] Connection reset by peer
-Department,Department of Government Information,dgi.gov.lk,valid,
-Department,Department of Government Printing,documents.gov.lk,valid,
-Department,Department of Excise,excise.gov.lk,valid,
-Department,Department of National Budget,nationalbudget.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Department,Department of Meteorology,meteo.gov.lk,valid,
-Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Department,Department of Forest Conservation,forestdept.gov.lk,expired,unable to get local issuer certificate
-Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,"[Errno 8] nodename nor servname provided, or not known"
-Department,Department of Examinations,doenets.lk,valid,
-Department,Department of Archaeology,archaeology.gov.lk,valid,
-Department,Department of Samurdhi,samurdhi.gov.lk,valid,
-Department,LSF,opensource.lk,invalid,"Hostname mismatch, certificate is not valid for 'opensource.lk'."
-Expired,Expired SSL,expired.badssl.com,unreachable,[Errno 54] Connection reset by peer
-Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate
-Wrong Host,Wrong Host SSL,wrong.host.badssl.com,invalid,"Hostname mismatch, certificate is not valid for 'wrong.host.badssl.com'."
-Never SSL,Never SSL,neverssl.com,valid,
+Type,Institution Name,URL,ssl_status,ssl_error,footer_vendor,footer_vendor_error
+Ministry,Ministry of Defence,defence.lk,valid,,unknown,
+Ministry,"Ministry of Finance, Planning and Economic Development",treasury.gov.lk,valid,,found,elysiancrest.com
+Ministry,Ministry of Digital Economy,midec.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,"Ministry of Foreign Affairs, Foreign Employment and Tourism",mfa.gov.lk,valid,,unknown,
+Ministry,"Ministry of Education, Higher Education and Vocational Education",moe.gov.lk,valid,,unknown,
+Ministry,"Ministry of Agriculture, Livestock, Land and Irrigation",agrimin.gov.lk,valid,,found,Ones and Zeros
+Ministry,"Ministry of Transport, Highways and Urban Development",transport.gov.lk,valid,,unknown,
+Ministry,"Ministry of Public Administration, Provincial Councils and Local Government",pubad.gov.lk,valid,,unknown,
+Ministry,Ministry of Health and Mass Media,health.gov.lk,valid,,unknown,
+Ministry,Ministry of Justice and National Integration,moj.gov.lk,valid,,found,Procons Infotech
+Ministry,Ministry of Public Security and Parliamentary Affairs,pubsec.gov.lk,valid,,unknown,
+Ministry,Ministry of Industry and Entrepreneurship Development,industry.gov.lk,valid,,unknown,
+Ministry,"Ministry of Trade, Commerce, Food Security and Cooperative Development",ment.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,Ministry of Environment,env.gov.lk,valid,,unknown,
+Ministry,Ministry of Energy,energy.gov.lk,valid,,unknown,
+Ministry,"Ministry of Housing, Construction and Water Supply",housingmin.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,Ministry of Labor,labourmin.gov.lk,valid,,unknown,
+Ministry,Ministry of Plantation and Community Infrastructure,plantationindustries.gov.lk,invalid,"Hostname mismatch, certificate is not valid for 'plantationindustries.gov.lk'.",found,University of Kelaniya
+Ministry,Ministry of Ports and Civil Aviation,portaviation.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,"Ministry of Fisheries, Aquatic and Ocean Resources",fisheries.gov.lk,valid,,self_built,IT UNIT Last Update
+Ministry,Ministry of Women and Child Affairs,childwomen.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,"Ministry of Rural Development, Social Security and Community Empowerment",socialservices.gov.lk,valid,,unknown,
+Ministry,"Ministry of Buddhasasana, Religious and Cultural Affairs",mbra.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,Ministry of Youth Affairs and Sports,sportsmin.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Ministry,Ministry of Science and Technology,most.gov.lk,valid,,unknown,
+Department,Department of Immigration and Emigration,immigration.gov.lk,valid,,unknown,
+Department,Department of Motor Traffic,dmt.gov.lk,valid,,found,Procons Infotech & gic.gov.lk
+Department,Department of Registration of Persons,drp.gov.lk,valid,,unknown,
+Department,Department of Registrar General,rgd.gov.lk,valid,,unknown,
+Department,Department of Inland Revenue,ird.gov.lk,valid,,unreachable,Failed to fetch page
+Department,Sri Lanka Customs,customs.gov.lk,valid,,unknown,
+Department,Department of Pensions,pensions.gov.lk,valid,,found,Procons Infotech
+Department,Department of Census and Statistics,statistics.gov.lk,unreachable,[WinError 10054] An existing connection was forcibly closed by the remote host,unreachable,Failed to fetch page
+Department,Department of Government Information,dgi.gov.lk,valid,,unknown,
+Department,Department of Government Printing,documents.gov.lk,valid,,unknown,
+Department,Department of Excise,excise.gov.lk,valid,,found,LankaCom
+Department,Department of National Budget,nationalbudget.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Department,Department of Meteorology,meteo.gov.lk,valid,,unknown,
+Department,Department of Wildlife Conservation,dwc.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Department,Department of Forest Conservation,forestdept.gov.lk,expired,certificate has expired,found,icta.lk & Procons Infotech
+Department,Department of National Zoological Gardens,colombozoo.gov.lk,unreachable,[Errno 11001] getaddrinfo failed,unreachable,Failed to fetch page
+Department,Department of Examinations,doenets.lk,valid,,unknown,
+Department,Department of Archaeology,archaeology.gov.lk,valid,,found,University of Kelaniya
+Department,Department of Samurdhi,samurdhi.gov.lk,valid,,unknown,
+Department,LSF,opensource.lk,valid,,unknown,
+Expired,Expired SSL,expired.badssl.com,expired,certificate has expired,unknown,
+Self-Signed,Self-Signed SSL,self-signed.badssl.com,invalid,self-signed certificate,unknown,
+Wrong Host,Wrong Host SSL,wrong.host.badssl.com,invalid,"Hostname mismatch, certificate is not valid for 'wrong.host.badssl.com'.",unknown,
+Never SSL,Never SSL,neverssl.com,valid,,unreachable,Failed to fetch page

src/websitescorecard/checks/__init__.py

@@ -6,11 +6,13 @@
 
 from websitescorecard.checks.base import Check
 from websitescorecard.checks.ssl import SSLCheck
+from websitescorecard.checks.footer_vendor import FooterVendorCheck
 
 CheckFactory = Callable[[], Check]
 
 CHECK_REGISTRY: dict[str, CheckFactory] = {
     "ssl": SSLCheck,
+    "footer_vendor": FooterVendorCheck,
 }