diff --git a/.gitignore b/.gitignore index 23b99e089..0845fb45a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ bibliovenv/ Bibenv/ -.idea/ \ No newline at end of file +.idea/ +venv/ \ No newline at end of file diff --git a/execution_evidence.ipynb b/execution_evidence.ipynb new file mode 100644 index 000000000..aae868210 --- /dev/null +++ b/execution_evidence.ipynb @@ -0,0 +1,811 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fa25a520", + "metadata": {}, + "source": [ + "# ETL Pipeline — Execution Evidence\n", + "Bibliometrix-Python ETL Exam — [Your Name/Group]\n", + "\n", + "This notebook documents the standardization pipeline (`get_data.py`,\n", + "`validation.py`) tested against real exports from Web of Science and PubMed.\n", + "\n", + "## 1. Architecture Overview\n", + "- **Extract**: reuses `parsers.py` (WoS, PubMed, Cochrane) and pandas readers (Scopus CSV, Dimensions XLSX) already present in the repo.\n", + "- **Transform**: reuses `format_functions.py`'s per-column formatters, which map each source's proprietary columns to the WoS standard schema.\n", + "- **Validate**: new module `validation.py` — enforces the type contract (list[str] for multi-value fields, str for scalars, no NaN/None).\n", + "- **Load**: new function `get_data.py` — orchestrates Extract → Transform → Validate → SR generation → `df.set()`, wired to the \"Start\" button.\n", + "- **SR (Short Reference)**: reused from existing `metaTagExtraction(df, Field=\"SR\")` in `metatagextraction.py`, not reimplemented." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "424cd8cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully processed: sample_wos.txt\n", + "Successfully processed 1 files.\n", + "Shape: (2, 34)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABAUAU_UNAU1_UNBPEPCRC1DBDE...RPSCSNSOSRTCTIUTVLAF
0This paper presents a bibliometric analysis of...[Smith J., Doe A.][]100115[Aria M, 2017, J INFORMETR, V11, P959, DOI 10....[Univ Naples Federico II, Dept Elect Engn & In...Web_of_Science[bibliometrics, open science, research evaluat......Smith, J (corresponding author), Univ Naples F...[]JOURNAL OF INFORMETRICSSmith J., 2022, JOURNAL OF INFORMETRICS12A bibliometric analysis of open science practicesWOS:000800000001[16][]
1We apply science mapping techniques, including...[Rossi M., Bianchi L., Verdi P.][]20012030[Aria M, 2017, J INFORMETR, V11, P959, DOI 10....[Univ Naples Federico II, Naples, Italy, Univ ...Web_of_Science[science mapping, co-citation analysis, themat......Rossi, M (corresponding author), Univ Naples F...[]SCIENTOMETRICSRossi M., 2023, SCIENTOMETRICS5Mapping the evolution of scientometric researc...WOS:000800000002[128][]
\n", + "

2 rows × 34 columns

\n", + "
" + ], + "text/plain": [ + " AB \\\n", + "0 This paper presents a bibliometric analysis of... \n", + "1 We apply science mapping techniques, including... \n", + "\n", + " AU AU_UN AU1_UN BP EP \\\n", + "0 [Smith J., Doe A.] [] 100 115 \n", + "1 [Rossi M., Bianchi L., Verdi P.] [] 2001 2030 \n", + "\n", + " CR \\\n", + "0 [Aria M, 2017, J INFORMETR, V11, P959, DOI 10.... \n", + "1 [Aria M, 2017, J INFORMETR, V11, P959, DOI 10.... \n", + "\n", + " C1 DB \\\n", + "0 [Univ Naples Federico II, Dept Elect Engn & In... Web_of_Science \n", + "1 [Univ Naples Federico II, Naples, Italy, Univ ... Web_of_Science \n", + "\n", + " DE ... \\\n", + "0 [bibliometrics, open science, research evaluat... ... \n", + "1 [science mapping, co-citation analysis, themat... ... \n", + "\n", + " RP SC SN \\\n", + "0 Smith, J (corresponding author), Univ Naples F... [] \n", + "1 Rossi, M (corresponding author), Univ Naples F... [] \n", + "\n", + " SO SR TC \\\n", + "0 JOURNAL OF INFORMETRICS Smith J., 2022, JOURNAL OF INFORMETRICS 12 \n", + "1 SCIENTOMETRICS Rossi M., 2023, SCIENTOMETRICS 5 \n", + "\n", + " TI UT VL \\\n", + "0 A bibliometric analysis of open science practices WOS:000800000001 [16] \n", + "1 Mapping the evolution of scientometric researc... WOS:000800000002 [128] \n", + "\n", + " AF \n", + "0 [] \n", + "1 [] \n", + "\n", + "[2 rows x 34 columns]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import sys\n", + "sys.path.append(\".\")\n", + "\n", + "from www.services.format_functions import process_multiple_files\n", + "from www.services.validation import validate_standardized_df, coerce_to_contract\n", + "from www.services.metatagextraction import metaTagExtraction\n", + "import pandas as pd, json\n", + "from shiny import reactive\n", + "\n", + "# Simulate a file upload for WoS\n", + "file_info = [{\"datapath\": \"sample_wos.txt\", \"name\": \"sample_wos.txt\"}]\n", + "json_data = process_multiple_files(file_info, \"wos\", \"surname\")\n", + "records = json.loads(json_data)\n", + "df_wos = pd.DataFrame(records)\n", + "\n", + "for col in (\"AU\", \"AF\"):\n", + " if col not in df_wos.columns:\n", + " df_wos[col] = [[] for _ in range(len(df_wos))]\n", + "\n", + "print(\"Shape:\", df_wos.shape)\n", + "df_wos.head()" + ] + }, + { + "cell_type": "markdown", + "id": "bb738365", + "metadata": {}, + "source": [ + "## 2. Validation" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "473d1766", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'is_valid': False, 'missing_columns': [], 'type_errors': {'FU': [0, 1], 'TC': [0, 1]}, 'null_errors': {'VL': [0, 1]}, 'row_count': 2, 'column_count': 34}\n" + ] + } + ], + "source": [ + "report = validate_standardized_df(df_wos, strict=False)\n", + "print(report)" + ] + }, + { + "cell_type": "markdown", + "id": "fad58faa", + "metadata": {}, + "source": [ + "Validation caught real type issues (TC as string instead of int, VL not fully\n", + "unwrapped from list). Per Phase 5 of the pipeline, `coerce_to_contract()` is\n", + "applied automatically in `get_data.py` when this happens. Demonstrating that\n", + "auto-fix here:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9f55f01d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'is_valid': True, 'missing_columns': [], 'type_errors': {}, 'null_errors': {}, 'row_count': 2, 'column_count': 34}\n" + ] + } + ], + "source": [ + "df_wos = coerce_to_contract(df_wos)\n", + "report_after = validate_standardized_df(df_wos, strict=False)\n", + "print(report_after)" + ] + }, + { + "cell_type": "markdown", + "id": "a1fe9c3b", + "metadata": {}, + "source": [ + "## 3. SR / SR_FULL Generation (reusing existing metaTagExtraction)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "05928e2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SRSR_FULL
0Smith J., 2022, JOURNAL OF INFORMETRICSSmith J., 2022, JOURNAL OF INFORMETRICS
1Rossi M., 2023, SCIENTOMETRICSRossi M., 2023, SCIENTOMETRICS
\n", + "
" + ], + "text/plain": [ + " SR \\\n", + "0 Smith J., 2022, JOURNAL OF INFORMETRICS \n", + "1 Rossi M., 2023, SCIENTOMETRICS \n", + "\n", + " SR_FULL \n", + "0 Smith J., 2022, JOURNAL OF INFORMETRICS \n", + "1 Rossi M., 2023, SCIENTOMETRICS " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with reactive.isolate():\n", + " holder = reactive.Value(df_wos)\n", + " holder = metaTagExtraction(holder, Field=\"SR\")\n", + " df_wos = holder.get()\n", + "\n", + "df_wos[[\"SR\", \"SR_FULL\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "efd5fc6b", + "metadata": {}, + "source": [ + "## 4. Repeat for a real PubMed export" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "44f796dc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Successfully processed: pubmed-bibliometr-set.txt\n", + "Successfully processed 1 files.\n", + "Shape: (10, 34)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ABAUAU_UNAU1_UNBPEPCRC1DBDE...RPSCSNSOSRTCTIUTVLAF
0Bibliometrics is the study of academic publish...[Ninkov A, Frank JR, Maggio LA][University of Ottawa, University of Ottawa, U...University of Ottawa173176[][School of Information Studies, University of ...PubMed[Bibliometrics, Humans, Publications, Publishing]...2212-277X (Electronic) 2212-2761 (Print) 2212-...Perspectives on medical educationNinkov A, 2022, Perspect Med Educ0Bibliometrics: Methods for studying academic p...3491402711[]
1As one of the emerging industries, feed indust...[Wang Y, Zheng C, Li H, Wu H][National Science Library, University of Chine...National Science Library241249[][Discipline Consulting Service, National Scien...PubMed[Animals, Bibliometrics, Cluster Analysis]...1872-2075 (Electronic) 1000-3061 (Linking)Sheng wu gong cheng xue bao = Chinese journal ...Wang Y, 2020, Sheng Wu Gong Cheng Xue Bao0[Bibliometrics evaluation of biological feed b...3214799636[]
2Polystyrene plastic pollution poses a pressing...[Tan Y, Ji L, Mo Y, Huang H, Lei X][Hengyang Medical School, Hengyang Medical S...Hengyang Medical School465478[][Clinical Anatomy and Reproductive Medicine Ap...PubMed[Bibliometrics, Humans, Polystyrenes, Infertil......1477-0393 (Electronic) 0748-2337 (Linking)Toxicology and industrial healthTan Y, 2024, Toxicol Ind Health0Bibliometrics analysis of hotspots research on...3880501540[]
3BACKGROUND: Alopecia areata (AA) is an autoimm...[Luo WR, Shen G, Yang LH, Zhu XH][The Second Affiliated Hospital (Changzheng H...The Second Affiliated Hospital (Changzheng Ho...4258[][Department of Plastic and Reconstructive Surg...PubMed[Humans, United States, Alopecia Areata/drug t......1421-9832 (Electronic) 1018-8665 (Linking)Dermatology (Basel, Switzerland)Luo WR, 2024, Dermatology0A Bibliometrics of the Treatment of Alopecia A...37939681240[]
4PURPOSE: Bibliometric analyses have gained pop...[Hani U, Chen JW, Holland C, McGirt MJ, Kim PK...[University of Vanderbilt, University of Vande...2533[][Carolina Neurosurgery and Spine Associates, 2...PubMed[Humans, Scoliosis/surgery, Technology, Biblio......2212-1358 (Electronic) 2212-134X (Linking)Spine deformityHani U, 2024, Spine Deform0Patent bibliometrics in spinal deformity: the ...3784560012[]
\n", + "

5 rows × 34 columns

\n", + "
" + ], + "text/plain": [ + " AB \\\n", + "0 Bibliometrics is the study of academic publish... \n", + "1 As one of the emerging industries, feed indust... \n", + "2 Polystyrene plastic pollution poses a pressing... \n", + "3 BACKGROUND: Alopecia areata (AA) is an autoimm... \n", + "4 PURPOSE: Bibliometric analyses have gained pop... \n", + "\n", + " AU \\\n", + "0 [Ninkov A, Frank JR, Maggio LA] \n", + "1 [Wang Y, Zheng C, Li H, Wu H] \n", + "2 [Tan Y, Ji L, Mo Y, Huang H, Lei X] \n", + "3 [Luo WR, Shen G, Yang LH, Zhu XH] \n", + "4 [Hani U, Chen JW, Holland C, McGirt MJ, Kim PK... \n", + "\n", + " AU_UN \\\n", + "0 [University of Ottawa, University of Ottawa, U... \n", + "1 [National Science Library, University of Chine... \n", + "2 [Hengyang Medical School, Hengyang Medical S... \n", + "3 [The Second Affiliated Hospital (Changzheng H... \n", + "4 [University of Vanderbilt, University of Vande... \n", + "\n", + " AU1_UN BP EP CR \\\n", + "0 University of Ottawa 173 176 [] \n", + "1 National Science Library 241 249 [] \n", + "2 Hengyang Medical School 465 478 [] \n", + "3 The Second Affiliated Hospital (Changzheng Ho... 42 58 [] \n", + "4 25 33 [] \n", + "\n", + " C1 DB \\\n", + "0 [School of Information Studies, University of ... PubMed \n", + "1 [Discipline Consulting Service, National Scien... PubMed \n", + "2 [Clinical Anatomy and Reproductive Medicine Ap... PubMed \n", + "3 [Department of Plastic and Reconstructive Surg... PubMed \n", + "4 [Carolina Neurosurgery and Spine Associates, 2... PubMed \n", + "\n", + " DE ... RP SC \\\n", + "0 [Bibliometrics, Humans, Publications, Publishing] ... \n", + "1 [Animals, Bibliometrics, Cluster Analysis] ... \n", + "2 [Bibliometrics, Humans, Polystyrenes, Infertil... ... \n", + "3 [Humans, United States, Alopecia Areata/drug t... ... \n", + "4 [Humans, Scoliosis/surgery, Technology, Biblio... ... \n", + "\n", + " SN \\\n", + "0 2212-277X (Electronic) 2212-2761 (Print) 2212-... \n", + "1 1872-2075 (Electronic) 1000-3061 (Linking) \n", + "2 1477-0393 (Electronic) 0748-2337 (Linking) \n", + "3 1421-9832 (Electronic) 1018-8665 (Linking) \n", + "4 2212-1358 (Electronic) 2212-134X (Linking) \n", + "\n", + " SO \\\n", + "0 Perspectives on medical education \n", + "1 Sheng wu gong cheng xue bao = Chinese journal ... \n", + "2 Toxicology and industrial health \n", + "3 Dermatology (Basel, Switzerland) \n", + "4 Spine deformity \n", + "\n", + " SR TC \\\n", + "0 Ninkov A, 2022, Perspect Med Educ 0 \n", + "1 Wang Y, 2020, Sheng Wu Gong Cheng Xue Bao 0 \n", + "2 Tan Y, 2024, Toxicol Ind Health 0 \n", + "3 Luo WR, 2024, Dermatology 0 \n", + "4 Hani U, 2024, Spine Deform 0 \n", + "\n", + " TI UT VL AF \n", + "0 Bibliometrics: Methods for studying academic p... 34914027 11 [] \n", + "1 [Bibliometrics evaluation of biological feed b... 32147996 36 [] \n", + "2 Bibliometrics analysis of hotspots research on... 38805015 40 [] \n", + "3 A Bibliometrics of the Treatment of Alopecia A... 37939681 240 [] \n", + "4 Patent bibliometrics in spinal deformity: the ... 37845600 12 [] \n", + "\n", + "[5 rows x 34 columns]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "file_info_pm = [{\"datapath\": \"pubmed-bibliometr-set.txt\", \"name\": \"pubmed-bibliometr-set.txt\"}]\n", + "json_data_pm = process_multiple_files(file_info_pm, \"pubmed\", \"surname\")\n", + "records_pm = json.loads(json_data_pm)\n", + "df_pubmed = pd.DataFrame(records_pm)\n", + "\n", + "for col in (\"AU\", \"AF\"):\n", + " if col not in df_pubmed.columns:\n", + " df_pubmed[col] = [[] for _ in range(len(df_pubmed))]\n", + "\n", + "print(\"Shape:\", df_pubmed.shape)\n", + "df_pubmed.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "1def80b6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Before coercion: {'is_valid': False, 'missing_columns': [], 'type_errors': {'OA': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'SC': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, 'null_errors': {}, 'row_count': 10, 'column_count': 34}\n", + "After coercion: {'is_valid': True, 'missing_columns': [], 'type_errors': {}, 'null_errors': {}, 'row_count': 10, 'column_count': 34}\n" + ] + } + ], + "source": [ + "report_pm = validate_standardized_df(df_pubmed, strict=False)\n", + "print(\"Before coercion:\", report_pm)\n", + "\n", + "df_pubmed = coerce_to_contract(df_pubmed)\n", + "report_pm_after = validate_standardized_df(df_pubmed, strict=False)\n", + "print(\"After coercion:\", report_pm_after)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "57e513c1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SRSR_FULL
0Ninkov A, 2022, Perspect Med EducNinkov A, 2022, Perspect Med Educ
1Wang Y, 2020, Sheng Wu Gong Cheng Xue BaoWang Y, 2020, Sheng Wu Gong Cheng Xue Bao
2Tan Y, 2024, Toxicol Ind HealthTan Y, 2024, Toxicol Ind Health
3Luo WR, 2024, DermatologyLuo WR, 2024, Dermatology
4Hani U, 2024, Spine DeformHani U, 2024, Spine Deform
\n", + "
" + ], + "text/plain": [ + " SR \\\n", + "0 Ninkov A, 2022, Perspect Med Educ \n", + "1 Wang Y, 2020, Sheng Wu Gong Cheng Xue Bao \n", + "2 Tan Y, 2024, Toxicol Ind Health \n", + "3 Luo WR, 2024, Dermatology \n", + "4 Hani U, 2024, Spine Deform \n", + "\n", + " SR_FULL \n", + "0 Ninkov A, 2022, Perspect Med Educ \n", + "1 Wang Y, 2020, Sheng Wu Gong Cheng Xue Bao \n", + "2 Tan Y, 2024, Toxicol Ind Health \n", + "3 Luo WR, 2024, Dermatology \n", + "4 Hani U, 2024, Spine Deform " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with reactive.isolate():\n", + " holder_pm = reactive.Value(df_pubmed)\n", + " holder_pm = metaTagExtraction(holder_pm, Field=\"SR\")\n", + " df_pubmed = holder_pm.get()\n", + "\n", + "df_pubmed[[\"SR\", \"SR_FULL\"]].head()" + ] + }, + { + "cell_type": "markdown", + "id": "e4d9f236", + "metadata": {}, + "source": [ + "## 5. Dashboard Validation (Shiny UI)\n", + "Screenshots below show the standardized data successfully powering\n", + "analysis functions in the live Bibliometrix-Python dashboard, across\n", + "two different bibliographic databases.\n", + "\n", + "### Web of Science — Main Information\n", + "![WoS Main Info](screenshots/wos_main_info.png)\n", + "\n", + "### PubMed — Most Frequent Words\n", + "![PubMed word freq](screenshots/pubmed_word_freq.png)\n", + "\n", + "### PubMed — Co-occurrence Network\n", + "![PubMed co-occurrence](screenshots/pubmed_cooccurrence.png)" + ] + }, + { + "cell_type": "markdown", + "id": "743c0cbe", + "metadata": {}, + "source": [ + "## 6. Bugs Found and Patched\n", + "\n", + "### Bug 1 — Missing AU/AF column\n", + "**File:** `www/services/format_functions.py`, `process_single_file()`\n", + "**Issue:** deletes AU or AF entirely based on author-format selection, instead of leaving an empty list. Violates the \"always-present column\" contract.\n", + "**Fix:** `get_data.py` restores the dropped column as `[[] for _ in range(len(df))]` after parsing.\n", + "\n", + "### Bug 2 — PY treated as numeric\n", + "**File:** `functions/get_maininformations.py`\n", + "**Issue:** `data[\"PY\"].max() - data[\"PY\"].min()` assumes int, but standardized schema stores PY as str per spec.\n", + "**Fix:** cast `data[\"PY\"]` to int locally at the top of the function, without mutating the shared reactive DataFrame.\n", + "\n", + "### Bug 3 — metaTagExtraction call discarded prior calculations\n", + "**File:** `functions/get_maininformations.py`\n", + "**Issue:** `data = df.get()` after `metaTagExtraction()` wiped out Min_Year/Max_Year/CAGR/PY-cast computed earlier.\n", + "**Fix:** merge only the new `AU_CO` column into the existing local `data`, instead of reassigning the whole DataFrame.\n", + "\n", + "### Bug 4 — Inconsistent typing across sources (TC, VL, OA, SC)\n", + "**Files:** `www/services/format_functions.py` (various `format_xx_column` functions)\n", + "**Issue:** `TC` returned as string instead of int for WoS; `VL` not fully unwrapped from its list container; PubMed leaves `OA`/`SC` in a non-list form.\n", + "**Fix:** Handled defensively in the Load phase by `coerce_to_contract()` (validation.py), per Phase 5 of the pipeline design — logged here as a known upstream inconsistency rather than patched at the source.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv (3.11.9)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/functions/get_maininformations.py b/functions/get_maininformations.py index 97443abdb..58064d192 100644 --- a/functions/get_maininformations.py +++ b/functions/get_maininformations.py @@ -12,7 +12,13 @@ def get_main_informations(df, log=False): Returns: A DataFrame with additional columns for filters and metrics. """ - data = df.get() + data = df.get().copy() + + # PY is stored as a string in the standardized schema (per spec: "str + # (or int)"), but this function performs arithmetic on it (max/min, + # CAGR, document age). Cast to int here, on a local copy, so the + # reactively-stored DataFrame elsewhere in the app is left untouched. + data["PY"] = pd.to_numeric(data["PY"], errors="coerce").fillna(0).astype(int) #### Min and Max Year #### start_time = time.time() @@ -97,9 +103,14 @@ def count_authors(entry): start_time = time.time() # Ensure the 'AU_CO' column exists if "AU_CO" not in data.columns: - # Extract the required metadata + # Extract the required metadata. + # NOTE: do NOT reassign `data = df.get()` here — that would discard + # every column already computed in this function (Min_Year, + # Max_Year, CAGR, the int-cast PY, etc.), since it fetches a fresh + # copy from the reactive store rather than the local working copy. + # Merge in only the newly-derived AU_CO column instead. df = metaTagExtraction(df, "AU_CO") - data = df.get() + data["AU_CO"] = df.get()["AU_CO"] # Calculate "Country_Count" with a vectorized function data["Country_Count"] = data["AU_CO"].apply(lambda x: len(set(x))) diff --git a/pubmed-bibliometr-set.txt b/pubmed-bibliometr-set.txt new file mode 100644 index 000000000..9a0546616 --- /dev/null +++ b/pubmed-bibliometr-set.txt @@ -0,0 +1,892 @@ +PMID- 34914027 +OWN - NLM +STAT- MEDLINE +DCOM- 20220630 +LR - 20230207 +IS - 2212-277X (Electronic) +IS - 2212-2761 (Print) +IS - 2212-2761 (Linking) +VI - 11 +IP - 3 +DP - 2022 Jun +TI - Bibliometrics: Methods for studying academic publishing. +PG - 173-176 +LID - 10.1007/s40037-021-00695-4 [doi] +AB - Bibliometrics is the study of academic publishing that uses statistics to + describe publishing trends and to highlight relationships between published + works. Likened to epidemiology, researchers seek to answer questions about + a field based on data about publications (e.g., authors, topics, funding) in the + same way that an epidemiologist queries patient data to understand the health of + a population. In this Eye Opener, the authors introduce bibliometrics and define + its key terminology and concepts, including relational and evaluative + bibliometrics. Readers are introduced to common bibliometric methods and their + related strengths and weaknesses. The authors provide examples of bibliometrics + applied in health professions education and propose potential future research + directions. Health professions educators are consumers of bibliometric reports + and can adopt its methodologies for future studies. +CI - © 2021. The Author(s). +FAU - Ninkov, Anton +AU - Ninkov A +AUID- ORCID: 0000-0002-8276-7656 +AD - School of Information Studies, University of Ottawa, Ottawa, Ontario, Canada. + aninkov@uottawa.ca. +FAU - Frank, Jason R +AU - Frank JR +AUID- ORCID: 0000-0002-6076-0146 +AD - Specialty Education, Royal College of Physicians and Surgeons of Canada, Ottawa, + Ontario, Canada. +AD - Department of Emergency Medicine, University of Ottawa, Ottawa, Ontario, Canada. +FAU - Maggio, Lauren A +AU - Maggio LA +AUID- ORCID: 0000-0002-2997-6133 +AD - Center for Health Professions Education and Department of Medicine, Uniformed + Services University, Bethesda, MD, USA. +LA - eng +PT - Journal Article +DEP - 20211216 +PL - Netherlands +TA - Perspect Med Educ +JT - Perspectives on medical education +JID - 101590643 +SB - IM +MH - *Bibliometrics +MH - Humans +MH - *Publications +MH - Publishing +PMC - PMC9240160 +OTO - NOTNLM +OT - Bibliometrics +OT - Information science +OT - Scholarly communication +COIS- A. Ninkov, J.R. Frank and L.A. Maggio declare that they have no competing + interests. +EDAT- 2021/12/17 06:00 +MHDA- 2022/07/01 06:00 +PMCR- 2021/12/16 +CRDT- 2021/12/16 12:30 +PHST- 2021/06/23 00:00 [received] +PHST- 2021/10/19 00:00 [accepted] +PHST- 2021/10/15 00:00 [revised] +PHST- 2021/12/17 06:00 [pubmed] +PHST- 2022/07/01 06:00 [medline] +PHST- 2021/12/16 12:30 [entrez] +PHST- 2021/12/16 00:00 [pmc-release] +AID - 10.1007/s40037-021-00695-4 [pii] +AID - 695 [pii] +AID - 10.1007/s40037-021-00695-4 [doi] +PST - ppublish +SO - Perspect Med Educ. 2022 Jun;11(3):173-176. doi: 10.1007/s40037-021-00695-4. Epub + 2021 Dec 16. + +PMID- 32147996 +OWN - NLM +STAT- MEDLINE +DCOM- 20200311 +LR - 20200311 +IS - 1872-2075 (Electronic) +IS - 1000-3061 (Linking) +VI - 36 +IP - 2 +DP - 2020 Feb 25 +TI - [Bibliometrics evaluation of biological feed based on patents]. +PG - 241-249 +LID - 10.13345/j.cjb.190223 [doi] +AB - As one of the emerging industries, feed industry not only effectively utilizes + existing agricultural resources but also provides a strong material foundation + and protection for animal food. In this article, status and trends of biological + feed in the world based on bibliometrics were analyzed. Issues including major + institutions, international cooperation, status, trends and frontiers were + analyzed. These co-countries map and keywords clustering analysis can reveal + co-countries and hot spots in this field. These analyses can help the researchers + get an overview of this field quickly and accurately. +FAU - Wang, Yang +AU - Wang Y +AD - Discipline Consulting Service, National Science Library, Chinese Academy of + Sciences, Beijing 100190, China. +AD - Department of Library, Information and Archives Management, University of Chinese + Academy of Sciences, Beijing 100190, China. +FAU - Zheng, Chunxiao +AU - Zheng C +AD - Discipline Consulting Service, National Science Library, Chinese Academy of + Sciences, Beijing 100190, China. +FAU - Li, Haiying +AU - Li H +AD - Discipline Consulting Service, National Science Library, Chinese Academy of + Sciences, Beijing 100190, China. +FAU - Wu, Hao +AU - Wu H +AD - Discipline Consulting Service, National Science Library, Chinese Academy of + Sciences, Beijing 100190, China. +AD - Department of Library, Information and Archives Management, University of Chinese + Academy of Sciences, Beijing 100190, China. +LA - chi +PT - Journal Article +PT - Review +PL - China +TA - Sheng Wu Gong Cheng Xue Bao +JT - Sheng wu gong cheng xue bao = Chinese journal of biotechnology +JID - 9426463 +SB - IM +MH - Animals +MH - *Bibliometrics +MH - Cluster Analysis +OTO - NOTNLM +OT - bibliometric analysis +OT - biological feed +OT - development trends analysis +OT - patents +EDAT- 2020/03/10 06:00 +MHDA- 2020/03/12 06:00 +CRDT- 2020/03/10 06:00 +PHST- 2020/03/10 06:00 [entrez] +PHST- 2020/03/10 06:00 [pubmed] +PHST- 2020/03/12 06:00 [medline] +AID - 10.13345/j.cjb.190223 [doi] +PST - ppublish +SO - Sheng Wu Gong Cheng Xue Bao. 2020 Feb 25;36(2):241-249. doi: + 10.13345/j.cjb.190223. + +PMID- 38805015 +OWN - NLM +STAT- MEDLINE +DCOM- 20240629 +LR - 20240701 +IS - 1477-0393 (Electronic) +IS - 0748-2337 (Linking) +VI - 40 +IP - 8 +DP - 2024 Aug +TI - Bibliometrics analysis of hotspots research on infertility syndromes and + polystyrene. +PG - 465-478 +LID - 10.1177/07482337241257274 [doi] +AB - Polystyrene plastic pollution poses a pressing environmental concern and + represents a significant risk factor for infertility. Despite this, a + comprehensive overview of the field remains scarce, with future trends largely + unknown. Bibliometrics, an applied mathematical and statistical method, offers a + means to analyze textual information across various levels, facilitating + quantitative assessments of all knowledge carriers and unveiling the nature and + developmental trajectories of a discipline. This study aimed to employ + bibliometric methods to scrutinize the current status and research hotspots + within the realm of polystyrene and infertility. Literature spanning from 1980 to + 2023 pertaining to polystyrene and infertility was retrieved from the core + database of Web of Science. Quantitative analyses were conducted utilizing + CiteSpace (version 5.7.R7), VOSviewer (version 1.6.18.0), and an online + literature analysis website (https://bibliometric.com/). The analysis visually + represented countries, institutions, authors, journals, and keywords within the + field. This study delved into the development history, knowledge structure, + research hotspots, and potential trends in the field, furnishing a macro + perspective for researchers. The investigation encompassed 267 articles published + across 120 journals by 1,352 authors affiliated with 417 institutions in 51 + countries, with these articles garnering 10,310 citations across 2,811 journals. + The top three countries contributing the most articles were China, the United + States, and Germany. In essence, the research hotspots primarily revolved around + metabolism, endocrinology, and immunity. Despite China's relatively recent entry + into this field, its rapid development is evident. However, the low citation + frequency suggests a need for improved article quality. +FAU - Tan, Yongpeng +AU - Tan Y +AD - Clinical Anatomy and Reproductive Medicine Application Institute, Hengyang + Medical School, University of South China, Hengyang, China. RINGGOLD: 34706 +FAU - Ji, Lin +AU - Ji L +AD - Reproductive Hospital of Guangxi Zhuang Autonomous Region, Nanning, China. +FAU - Mo, Yi +AU - Mo Y +AD - Reproductive Hospital of Guangxi Zhuang Autonomous Region, Nanning, China. +FAU - Huang, Hua +AU - Huang H +AUID- ORCID: 0009-0003-3777-3584 +AD - Reproductive Hospital of Guangxi Zhuang Autonomous Region, Nanning, China. +FAU - Lei, Xiaocan +AU - Lei X +AUID- ORCID: 0000-0001-7666-5082 +AD - Clinical Anatomy and Reproductive Medicine Application Institute, Hengyang + Medical School, University of South China, Hengyang, China. RINGGOLD: 34706 +LA - eng +PT - Journal Article +PT - Review +DEP - 20240528 +PL - England +TA - Toxicol Ind Health +JT - Toxicology and industrial health +JID - 8602702 +RN - 0 (Polystyrenes) +SB - IM +MH - *Bibliometrics +MH - Humans +MH - *Polystyrenes +MH - *Infertility/therapy/epidemiology +MH - Female +OTO - NOTNLM +OT - Polystyrene +OT - Web of Science +OT - bibliometrics +OT - country +OT - infertility +OT - quantitative analysis +COIS- Declaration of conflicting interestsThe author(s) declared no potential conflicts + of interest with respect to the research, authorship, and/or publication of this + article. +EDAT- 2024/05/28 12:44 +MHDA- 2024/06/29 15:42 +CRDT- 2024/05/28 10:43 +PHST- 2024/06/29 15:42 [medline] +PHST- 2024/05/28 12:44 [pubmed] +PHST- 2024/05/28 10:43 [entrez] +AID - 10.1177/07482337241257274 [doi] +PST - ppublish +SO - Toxicol Ind Health. 2024 Aug;40(8):465-478. doi: 10.1177/07482337241257274. Epub + 2024 May 28. + +PMID- 37939681 +OWN - NLM +STAT- MEDLINE +DCOM- 20240214 +LR - 20240214 +IS - 1421-9832 (Electronic) +IS - 1018-8665 (Linking) +VI - 240 +IP - 1 +DP - 2024 +TI - A Bibliometrics of the Treatment of Alopecia Areata in the Past Twenty Years. +PG - 42-58 +LID - 10.1159/000535043 [doi] +AB - BACKGROUND: Alopecia areata (AA) is an autoimmune disorder characterized by hair + loss on the scalp, face, and other body areas. Despite affecting approximately 2% + of the global population, there has been no previous bibliometric analysis + specifically focusing on AA treatment that can guide researchers in exploring + promising treatment options and directing future research efforts. SUMMARY: This + study conducted a bibliometric analysis of AA treatment research, encompassing + publications from 2003 to 2022. A total of 1,323 papers from 65 countries, + predominantly led by the USA and China, were included in the analysis. The number + of publications related to AA treatment showed a notable increase over the years. + Prominent research institutions included the University of Manchester, Icahn + School of Medicine at Mount Sinai, University of Miami, and Columbia University. + Among the journals, Dermatologic Therapy stood out as the most popular, while the + Journal of the American Academy of Dermatology appeared as the most frequently + co-cited publication. +CI - © 2023 S. Karger AG, Basel. +FAU - Luo, Wen-Rong +AU - Luo WR +AD - Department of Plastic and Reconstructive Surgery, The Second Affiliated Hospital + (Changzheng Hospital) of Naval Medical University, Shanghai, China, + 1281149166@qq.com. +FAU - Shen, Gan +AU - Shen G +AD - Department of Plastic and Reconstructive Surgery, The Second Affiliated Hospital + (Changzheng Hospital) of Naval Medical University, Shanghai, China. +FAU - Yang, Li-Hua +AU - Yang LH +AD - Department of Emergency, Naval Hospital of Eastern Theater, Zhoushan, China. +FAU - Zhu, Xiao-Hai +AU - Zhu XH +AD - Department of Plastic and Reconstructive Surgery, The Second Affiliated Hospital + (Changzheng Hospital) of Naval Medical University, Shanghai, China. +LA - eng +PT - Journal Article +PT - Review +DEP - 20231108 +PL - Switzerland +TA - Dermatology +JT - Dermatology (Basel, Switzerland) +JID - 9203244 +RN - Diffuse alopecia +SB - IM +MH - Humans +MH - United States +MH - *Alopecia Areata/drug therapy +MH - Bibliometrics +MH - Scalp +MH - China +OTO - NOTNLM +OT - Alopecia areata +OT - Bibliometrics +OT - Treatment +OT - Visualization +EDAT- 2023/11/09 00:41 +MHDA- 2024/02/13 06:45 +CRDT- 2023/11/08 18:24 +PHST- 2023/06/26 00:00 [received] +PHST- 2023/11/02 00:00 [accepted] +PHST- 2024/02/13 06:45 [medline] +PHST- 2023/11/09 00:41 [pubmed] +PHST- 2023/11/08 18:24 [entrez] +AID - 000535043 [pii] +AID - 10.1159/000535043 [doi] +PST - ppublish +SO - Dermatology. 2024;240(1):42-58. doi: 10.1159/000535043. Epub 2023 Nov 8. + +PMID- 37845600 +OWN - NLM +STAT- MEDLINE +DCOM- 20240108 +LR - 20240201 +IS - 2212-1358 (Electronic) +IS - 2212-134X (Linking) +VI - 12 +IP - 1 +DP - 2024 Jan +TI - Patent bibliometrics in spinal deformity: the first bibliometric analysis of + spinal deformity's technological literature. +PG - 25-33 +LID - 10.1007/s43390-023-00767-x [doi] +AB - PURPOSE: Bibliometric analyses have gained popularity for studying scientific + literature, but their application to evaluate technological literature (patents) + remains unexplored. We conducted a bibliometric analysis on the top 100 + most-cited patents in scoliosis surgery. METHODS: Multiple databases were queried + using The Lens to identify the top 100 scoliosis surgery patents, which were + selected based on forward patent citations. These patents were then categorized + into 8 groups based on technological descriptors and assessed based on various + factors including earliest priority date, year issued, and expiration status. + RESULTS: The top 100 most-cited patents included technology underlying + anterolateral tethering and distraction systems (n = 11), posterior tethering and + distraction systems (n = 23), posterior segmental bone anchor and rod engagement + systems (n = 29), interbody devices (n = 10), biological and electrophysiological + agents for scoliosis treatment and/or improved arthrodesis (n = 8), + intraoperative arthroplasty devices (n = 5), orthotic devices (n = 12), and + implantable devices for non-invasive, postoperative alterations of skeletal + alignment (n = 2). Seventy-five patents were expired, 21 are still active, and 4 + were listed as inactive. The late 1970s and early 2000s saw increased numbers of + patent filings. Demonstrated trends showed no meaningful correlation between + patent rank and earliest priority date (linear trendline y = 0.2648x - 477.27; + R(2) = 0.0114), while a very strong correlation was found between patent rank and + citations per year (power trendline y = 118.82x(--0.83); R(2) = 0.8983). + CONCLUSION: Patent bibliometric analyses in the field of spinal deformity surgery + provide a means to assess past advancements, better understand what it takes to + make a difference in the field, and to potentially facilitate the development of + innovative technologies in the future. The method described is a reliable and + reproducible technique for evaluating technological literature in our field. +CI - © 2023. The Author(s), under exclusive licence to Scoliosis Research Society. +FAU - Hani, Ummey +AU - Hani U +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. +FAU - Chen, Jeffrey Wu +AU - Chen JW +AD - School of Medicine, University of Vanderbilt, Nashville, TN, USA. +FAU - Holland, Christopher +AU - Holland C +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. +FAU - McGirt, Matthew J +AU - McGirt MJ +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. +FAU - Kim, Paul K +AU - Kim PK +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. +FAU - Chewning, Samuel +AU - Chewning S +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. +AD - School of Medicine, University of Vanderbilt, Nashville, TN, USA. +FAU - Bohl, Michael A +AU - Bohl MA +AD - Carolina Neurosurgery and Spine Associates, 225 Baldwin Avenue, Charlotte, NC, + 28204, USA. Michael.Bohl@cnsa.com. +AD - Spinefirst, Atrium Health, Charlotte, NC, 28203, USA. Michael.Bohl@cnsa.com. +LA - eng +PT - Journal Article +PT - Review +DEP - 20231016 +PL - England +TA - Spine Deform +JT - Spine deformity +JID - 101603979 +SB - IM +MH - Humans +MH - *Scoliosis/surgery +MH - Technology +MH - Bibliometrics +MH - Arthrodesis +OTO - NOTNLM +OT - Bibliometrics +OT - Deformity +OT - Patent +OT - Relevancy +OT - Spine surgery +EDAT- 2023/10/17 00:42 +MHDA- 2024/01/08 06:42 +CRDT- 2023/10/16 23:44 +PHST- 2023/07/17 00:00 [received] +PHST- 2023/09/13 00:00 [accepted] +PHST- 2024/01/08 06:42 [medline] +PHST- 2023/10/17 00:42 [pubmed] +PHST- 2023/10/16 23:44 [entrez] +AID - 10.1007/s43390-023-00767-x [pii] +AID - 10.1007/s43390-023-00767-x [doi] +PST - ppublish +SO - Spine Deform. 2024 Jan;12(1):25-33. doi: 10.1007/s43390-023-00767-x. Epub 2023 + Oct 16. + +PMID- 41574872 +OWN - NLM +STAT- MEDLINE +DCOM- 20260630 +LR - 20260630 +IS - 2245-1919 (Electronic) +IS - 2245-1919 (Linking) +VI - 73 +IP - 1 +DP - 2025 Dec 12 +TI - Five major challenges for medical bibliometrics. +LID - A09250723 [pii] +LID - 10.61409/A09250723 [doi] +AB - Bibliometrics, in terms of counting articles and books, has always existed. Since + 1955, however, additional parameters such as article citations, impact factors + and h-indexes have accompanied science - not least medical science. Today, + bibliometrics is big business where private companies sell all kinds of + bibliometric data. To handle these, it is essential to become aware of the + limitations and pitfalls in bibliometrics. Accordingly, the present review + describes five major challenges under the headings: Quality; Impact; + Co-authorship; Databases, and Fraud. +CI - Published under Open Access CC-BY-NC-BD 4.0. + https://creativecommons.org/licenses/by-nc-nd/4.0/. +FAU - Rehfeld, Jens F +AU - Rehfeld JF +AD - Department of Clinical Biochemistry, Copenhagen University Hospital - + Rigshospitalet. +LA - eng +PT - Journal Article +PT - Review +DEP - 20251212 +PL - Denmark +TA - Dan Med J +JT - Danish medical journal +JID - 101576205 +SB - IM +CIN - Dan Med J. 2026 Feb 10;73(3):1-2. PMID: 41773010 +MH - Humans +MH - *Bibliometrics +MH - Authorship +MH - Journal Impact Factor +MH - Fraud +EDAT- 2026/01/23 13:05 +MHDA- 2026/01/23 13:06 +CRDT- 2026/01/23 07:33 +PHST- 2026/01/23 13:06 [medline] +PHST- 2026/01/23 13:05 [pubmed] +PHST- 2026/01/23 07:33 [entrez] +AID - A09250723 [pii] +AID - 10.61409/A09250723 [doi] +PST - epublish +SO - Dan Med J. 2025 Dec 12;73(1):A09250723. doi: 10.61409/A09250723. + +PMID- 36584892 +OWN - NLM +STAT- MEDLINE +DCOM- 20230307 +LR - 20240614 +IS - 1878-8769 (Electronic) +IS - 1878-8750 (Linking) +VI - 171 +DP - 2023 Mar +TI - Review: Patent Bibliometrics in Cranial Neurosurgery: The First Bibliometric + Analysis of Neurosurgery's Technological Literature. +PG - 115-123 +LID - S1878-8750(22)01809-5 [pii] +LID - 10.1016/j.wneu.2022.12.103 [doi] +AB - BACKGROUND: Bibliometric analyses of the scientific literature have grown + increasingly popular in the past few decades. However, patent bibliometric + studies, evaluation of technological literature, have not yet been applied in + neurosurgery. OBJECTIVE: To perform a pilot patent bibliometric analysis of the + top 100 most cited patents in cranial neurosurgery. METHODS: The Lens was used to + query multiple databases, to select the top 100 cranial neurosurgical patents + based upon forward patent citations. These were organized into 9 categories based + on technological descriptors and were evaluated based on the earliest priority + date, year issued, and expiration status, among others. RESULTS: The top 100 most + cited patents included technology underlying 3D navigation (n = 31), pharmacology + and implants (n = 20), vascular occlusion (n = 5), craniotomy closure (n = 9), + focal lesioning and tissue resection (n = 8), brain and systemic cooling (n = 5), + neuroendoscopy (n = 8), neuromonitoring and stimulation (6), and technologies + improving surgeon performance (n = 8). Ninety-six patents were filed in the + United States, 72 were expired, 19 are still active, and 9 were listed as + inactive. The highest number of patents was applied for from the mid-1990s to the + mid-2000s. Demonstrated trends showed no meaningful correlation between patent + rank and earliest priority date (linear trendline y = 0.7107 x -1367.5; + R(2) = 0.0671), while a very strong correlation was found between patent rank and + citations per year (power trendline y = 127.93 x -1.094; R(2) = 0.8579). + CONCLUSIONS: Patent bibliometrics allow evaluation of neurosurgical advancements + from the past and enable subsequent development of cutting-edge technology in the + future. The described method is a reproducible and reliable technique for + evaluating our field's patent literature. +CI - Copyright © 2022 Elsevier Inc. All rights reserved. +FAU - Hani, Ummey +AU - Hani U +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - Mulvaney, Graham G +AU - Mulvaney GG +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - O'Brien, Matthew D +AU - O'Brien MD +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - Jernigan, Sarah +AU - Jernigan S +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - Kim, Paul +AU - Kim P +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - Holland, Christopher +AU - Holland C +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - McGirt, Matthew J +AU - McGirt MJ +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. +FAU - Bohl, Michael A +AU - Bohl MA +AD - Carolina Neurosurgery & Spine Associates, Charlotte, North Carolina, USA; + SpineFirst, Atrium Health, Charlotte, North Carolina, USA. Electronic address: + Michael.Bohl@cnsa.com. +LA - eng +PT - Journal Article +PT - Review +DEP - 20221227 +PL - United States +TA - World Neurosurg +JT - World neurosurgery +JID - 101528275 +SB - IM +MH - Humans +MH - United States +MH - *Neurosurgery +MH - Bibliometrics +MH - Neurosurgical Procedures +MH - Publications +MH - Technology +OTO - NOTNLM +OT - Bibliometrics +OT - Brain surgery +OT - Patent +OT - Relevancy +EDAT- 2022/12/31 06:00 +MHDA- 2023/03/08 06:00 +CRDT- 2022/12/30 19:15 +PHST- 2022/09/26 00:00 [received] +PHST- 2022/12/25 00:00 [accepted] +PHST- 2022/12/31 06:00 [pubmed] +PHST- 2023/03/08 06:00 [medline] +PHST- 2022/12/30 19:15 [entrez] +AID - S1878-8750(22)01809-5 [pii] +AID - 10.1016/j.wneu.2022.12.103 [doi] +PST - ppublish +SO - World Neurosurg. 2023 Mar;171:115-123. doi: 10.1016/j.wneu.2022.12.103. Epub 2022 + Dec 27. + +PMID- 36311729 +OWN - NLM +STAT- MEDLINE +DCOM- 20221101 +LR - 20221102 +IS - 1664-3224 (Electronic) +IS - 1664-3224 (Linking) +VI - 13 +DP - 2022 +TI - A bibliometric analysis of T cell and atherosclerosis. +PG - 948314 +LID - 10.3389/fimmu.2022.948314 [doi] +LID - 948314 +AB - Atherosclerosis (AS) is widespread and develops into circulatory system problems. + T cells play an essential regulatory role in AS occurrence and development. So + far, there is no bibliometric research on T cells and AS. To learn more about T + cell and AS development, 4,381 records were retrieved from Web of Science™ Core + Collection. Then, these records were scientometrically analyzed using CiteSpace + and VOSviewer in terms of spatiotemporal distribution, author distribution, + subject categories, topic distribution, references, and keywords. Our analysis + provides basic information on research in the field, demonstrates that the field + has stabilized over the past decade, and identifies potential partners for + interested researchers. Current research hotspots in this field mainly include + the inflammatory mechanism, immune mechanism, related diseases, and related + cytokines of AS. B cell, mortality, inhibition, and monocyte represent the + frontiers of research in this field, undergoing an explosive phase. We hope that + this work will provide new ideas for advancing the scientific research and + clinical application of T cell and AS. +CI - Copyright © 2022 Wei, Xu, Li, Shi, Zhang, You, Sun, Zhai and Hu. +FAU - Wei, Namin +AU - Wei N +AD - School of Chinese Materia Medica, Beijing University of Chinese Medicine, + Beijing, China. +FAU - Xu, Yan +AU - Xu Y +AD - School of Chinese Materia Medica, Beijing University of Chinese Medicine, + Beijing, China. +FAU - Li, Ya'nan +AU - Li Y +AD - School of Chinese Materia Medica, Beijing University of Chinese Medicine, + Beijing, China. +FAU - Shi, Jingjing +AU - Shi J +AD - Department of Cardiovascular Diseases, Guang'anmen Hospital, China Academy of + Chinese Medical Sciences, Beijing, China. +FAU - Zhang, Xuesong +AU - Zhang X +AD - Department of Cardiovascular Diseases, Guang'anmen Hospital, China Academy of + Chinese Medical Sciences, Beijing, China. +FAU - You, Yaping +AU - You Y +AD - Department of Cardiovascular Diseases, Guang'anmen Hospital, China Academy of + Chinese Medical Sciences, Beijing, China. +FAU - Sun, Qianqian +AU - Sun Q +AD - School of Chinese Materia Medica, Beijing University of Chinese Medicine, + Beijing, China. +FAU - Zhai, Huaqiang +AU - Zhai H +AD - School of Chinese Materia Medica, Beijing University of Chinese Medicine, + Beijing, China. +FAU - Hu, Yuanhui +AU - Hu Y +AD - Department of Cardiovascular Diseases, Guang'anmen Hospital, China Academy of + Chinese Medical Sciences, Beijing, China. +LA - eng +PT - Journal Article +DEP - 20221013 +PL - Switzerland +TA - Front Immunol +JT - Frontiers in immunology +JID - 101560960 +SB - IM +MH - Humans +MH - *Bibliometrics +MH - *Atherosclerosis +MH - T-Lymphocytes +PMC - PMC9606647 +OTO - NOTNLM +OT - CiteSpace +OT - T cell +OT - VOSviewer +OT - atherosclerosis +OT - bibliometrics +OT - visualization +COIS- The authors declare that the research was conducted in the absence of any + commercial or financial relationships that could be construed as a potential + conflict of interest. +EDAT- 2022/11/01 06:00 +MHDA- 2022/11/02 06:00 +PMCR- 2022/01/01 +CRDT- 2022/10/31 04:26 +PHST- 2022/05/19 00:00 [received] +PHST- 2022/09/30 00:00 [accepted] +PHST- 2022/10/31 04:26 [entrez] +PHST- 2022/11/01 06:00 [pubmed] +PHST- 2022/11/02 06:00 [medline] +PHST- 2022/01/01 00:00 [pmc-release] +AID - 10.3389/fimmu.2022.948314 [doi] +PST - epublish +SO - Front Immunol. 2022 Oct 13;13:948314. doi: 10.3389/fimmu.2022.948314. eCollection + 2022. + +PMID- 35935932 +OWN - NLM +STAT- MEDLINE +DCOM- 20221117 +LR - 20221117 +IS - 1664-3224 (Electronic) +IS - 1664-3224 (Linking) +VI - 13 +DP - 2022 +TI - Knowledge Mapping of Exosomes in Autoimmune Diseases: A Bibliometric Analysis + (2002-2021). +PG - 939433 +LID - 10.3389/fimmu.2022.939433 [doi] +LID - 939433 +AB - BACKGROUND: Autoimmune diseases (AIDs) are a class of chronic disabling diseases + characterized by inflammation and damage to muscles, joints, bones, and internal + organs. Recent studies have shown that much progress has been made in the + research of exosomes in AIDs. However, there is no bibliometric analysis in this + research field. This study aims to provide a comprehensive overview of the + knowledge structure and research hotspots of exosomes in AIDs through + bibliometrics. METHOD: Publications related to exosomes in AIDs from 2002 to 2021 + were searched on the web of science core collection (WoSCC) database. VOSviewers, + CiteSpace and R package "bibliometrix" were used to conduct this bibliometric + analysis. RESULTS: 312 articles from 48 countries led by China and the United + States were included. The number of publications related to exosomes in AIDs is + increasing year by year. Central South University, Sun Yat Sen University, + Tianjin Medical University and University of Pennsylvania are the main research + institutions. Frontiers in immunology is the most popular journal in this field, + and Journal of Immunology is the most co-cited journal. These publications come + from 473 authors among which Ilias Alevizos, Qianjin Lu, Wei Wei, Jim Xiang and + Ming Zhao had published the most papers and Clotilde Théry was co-cited most + often. Studying the mechanism of endogenous exosomes in the occurrence and + development of AIDs and the therapeutic strategy of exogenous exosomes in AIDs + are the main topics in this research field. "Mesenchymal stem cells", "microRNA", + "biomarkers", "immunomodulation", and "therapy" are the primary keywords of + emerging research hotspots. CONCLUSION: This is the first bibliometric study that + comprehensively summarizes the research trends and developments of exosomes in + AIDs. This information identifies recent research frontiers and hot directions, + which will provide a reference for scholars studying exosomes. +CI - Copyright © 2022 Wu, Gao, Kang, Wang, Niu, Liu and Zhang. +FAU - Wu, Fengping +AU - Wu F +AD - School of Basic Medical Sciences, Shanxi Medical University, Taiyuan, China. +FAU - Gao, Jinfang +AU - Gao J +AD - Department of Rheumatology, Shanxi Bethune Hospital, Shanxi Academy of Medical + Sciences, Tongji Shanxi Hospital, Third Hospital of Shanxi Medical University, + Taiyuan, China. +FAU - Kang, Jie +AU - Kang J +AD - Third Hospital of Shanxi Medical University, Shanxi Bethune Hospital, Shanxi + Academy of Medical Sciences, Tongji Shanxi Hospital, Taiyuan, China. +FAU - Wang, Xuexue +AU - Wang X +AD - Third Hospital of Shanxi Medical University, Shanxi Bethune Hospital, Shanxi + Academy of Medical Sciences, Tongji Shanxi Hospital, Taiyuan, China. +FAU - Niu, Qing +AU - Niu Q +AD - School of Basic Medical Sciences, Shanxi Medical University, Taiyuan, China. +FAU - Liu, Jiaxi +AU - Liu J +AD - Third Hospital of Shanxi Medical University, Shanxi Bethune Hospital, Shanxi + Academy of Medical Sciences, Tongji Shanxi Hospital, Taiyuan, China. +FAU - Zhang, Liyun +AU - Zhang L +AD - Department of Rheumatology, Shanxi Bethune Hospital, Shanxi Academy of Medical + Sciences, Tongji Shanxi Hospital, Third Hospital of Shanxi Medical University, + Taiyuan, China. +LA - eng +PT - Journal Article +PT - Research Support, Non-U.S. Gov't +DEP - 20220722 +PL - Switzerland +TA - Front Immunol +JT - Frontiers in immunology +JID - 101560960 +SB - IM +MH - Humans +MH - *Autoimmune Diseases +MH - Bibliometrics +MH - *Exosomes +MH - United States +PMC - PMC9353180 +OTO - NOTNLM +OT - CiteSpace +OT - VOSviewers +OT - autoimmune diseases +OT - bibliometrics +OT - exosomes +COIS- The authors declare that the research was conducted in the absence of any + commercial or financial relationships that could be construed as a potential + conflict of interest. +EDAT- 2022/08/09 06:00 +MHDA- 2022/08/10 06:00 +PMCR- 2022/01/01 +CRDT- 2022/08/08 03:41 +PHST- 2022/05/09 00:00 [received] +PHST- 2022/06/24 00:00 [accepted] +PHST- 2022/08/08 03:41 [entrez] +PHST- 2022/08/09 06:00 [pubmed] +PHST- 2022/08/10 06:00 [medline] +PHST- 2022/01/01 00:00 [pmc-release] +AID - 10.3389/fimmu.2022.939433 [doi] +PST - epublish +SO - Front Immunol. 2022 Jul 22;13:939433. doi: 10.3389/fimmu.2022.939433. eCollection + 2022. + +PMID- 39628489 +OWN - NLM +STAT- MEDLINE +DCOM- 20241204 +LR - 20241205 +IS - 1664-3224 (Electronic) +IS - 1664-3224 (Linking) +VI - 15 +DP - 2024 +TI - A bibliometric analysis of immunotherapy for atherosclerosis: trends and hotspots + prediction. +PG - 1493250 +LID - 10.3389/fimmu.2024.1493250 [doi] +LID - 1493250 +AB - INTRODUCTION: An increasing number of studies have demonstrated that + immunotherapy may play a significant role in treating Atherosclerosis and has + emerged as a promising therapy in this field. The aim of this study is to provide + a comprehensive perspective through bibliometric analysis and investigate the + existing hotspots and frontiers. METHODS: This study searched records from Web of + Science, PubMed, and Scopus from January 1, 1999, to May 27, 2023. By using + bibliometric software CiteSpace (6.3.R1) and VOSviewer (1.6.19), co-occurrence + analysis was used to count the frequency of co-occurrence of certain elements + (e.g., countries, regions, institutions, etc.), cluster analysis was used to + classify keywords, and burst analysis was used to identify research trends and + hotspots. RESULTS: The results showed that the number of annual publications has + grown in a fluctuating manner; the USA, China, and the Netherlands have the + highest numbers of publications, and the top three institutions are located in + the Netherlands, Sweden, and the USA. In addition, Nilsson J published the + highest number of papers; Ridker PM and his article "Anti-inflammatory Therapy + with Canakinumab for Atherosclerotic Disease" have played prominent roles. The + top four Journals with the highest numbers of publications are "Arteriosclerosis + Thrombosis and Vascular Biology", "Frontiers in Cardiovascular Medicine", + "Circulation" and "Vaccine". In addition, keyword analysis indicates that + inflammation, nanoparticles, adverse events associated with immune checkpoint + inhibitors, T cells and tumor necrosis factor will be future research hotspots. + DISCUSSION: This study provides a comprehensive bibliometric analysis of + immunotherapy in atherosclerosis, offering insights that advance scientific + understanding. It not only assists researchers in grasping the current hotspots + in this field but also reveals potential directions for future investigation. + Moreover, future studies can optimize immunotherapy strategies based on hotspot + predictions to decelerate the progression of atherosclerosis. +CI - Copyright © 2024 Wang, Pan and Jiang. +FAU - Wang, Jing-Hui +AU - Wang JH +AD - Department of Cardiovascular Medicine, The Second Affiliated Hospital of Nanchang + University, Nanchang, Jiangxi, China. +AD - Nanchang University Queen Mary School, Nanchang, Jiangxi, China. +FAU - Pan, Guan-Rui +AU - Pan GR +AD - Department of Cardiovascular Medicine, The Second Affiliated Hospital of Nanchang + University, Nanchang, Jiangxi, China. +AD - Nanchang University Queen Mary School, Nanchang, Jiangxi, China. +FAU - Jiang, Long +AU - Jiang L +AD - Department of Cardiovascular Medicine, The Second Affiliated Hospital of Nanchang + University, Nanchang, Jiangxi, China. +LA - eng +PT - Journal Article +PT - Systematic Review +DEP - 20241119 +PL - Switzerland +TA - Front Immunol +JT - Frontiers in immunology +JID - 101560960 +SB - IM +MH - *Bibliometrics +MH - Humans +MH - *Atherosclerosis/therapy/immunology +MH - *Immunotherapy/methods/trends +PMC - PMC11611808 +OTO - NOTNLM +OT - atherosclerosis +OT - bibliometrics +OT - hotspots +OT - immunotherapy +OT - trends +OT - visualization +COIS- The authors declare that the research was conducted in the absence of any + commercial or financial relationships that could be construed as a potential + conflict of interest. +EDAT- 2024/12/04 11:25 +MHDA- 2024/12/04 11:26 +PMCR- 2024/01/01 +CRDT- 2024/12/04 04:34 +PHST- 2024/09/08 00:00 [received] +PHST- 2024/11/04 00:00 [accepted] +PHST- 2024/12/04 11:26 [medline] +PHST- 2024/12/04 11:25 [pubmed] +PHST- 2024/12/04 04:34 [entrez] +PHST- 2024/01/01 00:00 [pmc-release] +AID - 10.3389/fimmu.2024.1493250 [doi] +PST - epublish +SO - Front Immunol. 2024 Nov 19;15:1493250. doi: 10.3389/fimmu.2024.1493250. + eCollection 2024. diff --git a/sample_wos.txt b/sample_wos.txt new file mode 100644 index 000000000..a182133b5 --- /dev/null +++ b/sample_wos.txt @@ -0,0 +1,66 @@ +FN Clarivate Analytics Web of Science +VR 1.0 +PT J +AU Smith, J. + Doe, A. +AF Smith, John + Doe, Anna +TI A bibliometric analysis of open science practices +SO JOURNAL OF INFORMETRICS +LA English +DT Article +DE bibliometrics; open science; research evaluation +ID SCIENTOMETRICS; RESEARCH IMPACT +AB This paper presents a bibliometric analysis of open science practices + across disciplines, highlighting trends in citation impact and + collaboration patterns over the last decade. +C1 [Smith, J.] Univ Naples Federico II, Dept Elect Engn & Informat Technol, Naples, Italy. + [Doe, A.] Univ Naples Federico II, Dept Econ & Stat, Naples, Italy. +RP Smith, J (corresponding author), Univ Naples Federico II, Dept Elect Engn & Informat Technol, Naples, Italy. +EM j.smith@unina.it +CR Aria M, 2017, J INFORMETR, V11, P959, DOI 10.1016/j.joi.2017.08.007 + Cuccurullo C, 2016, SCIENTOMETRICS, V108, P595, DOI 10.1007/s11192-016-1948-8 +TC 12 +PY 2022 +VL 16 +IS 2 +BP 100 +EP 115 +DI 10.1016/j.joi.2022.101300 +UT WOS:000800000001 +ER + +PT J +AU Rossi, M. + Bianchi, L. + Verdi, P. +AF Rossi, Marco + Bianchi, Laura + Verdi, Paolo +TI Mapping the evolution of scientometric research: a science mapping approach +SO SCIENTOMETRICS +LA English +DT Review +DE science mapping; co-citation analysis; thematic evolution +ID BIBLIOMETRIX; COCITATION ANALYSIS +AB We apply science mapping techniques, including co-citation and + co-word analysis, to trace the evolution of scientometric research + themes from 2000 to 2023. +C1 [Rossi, M.] Univ Naples Federico II, Naples, Italy. + [Bianchi, L.] Univ Naples Federico II, Naples, Italy. + [Verdi, P.] Univ Naples Federico II, Naples, Italy. +RP Rossi, M (corresponding author), Univ Naples Federico II, Naples, Italy. +EM m.rossi@unina.it +CR Aria M, 2017, J INFORMETR, V11, P959, DOI 10.1016/j.joi.2017.08.007 + Small H, 1973, J AM SOC INFORM SCI, V24, P265, DOI 10.1002/asi.4630240406 +TC 5 +PY 2023 +VL 128 +IS 4 +BP 2001 +EP 2030 +DI 10.1007/s11192-023-04700-1 +UT WOS:000800000002 +ER + +EF diff --git a/screenshots/pubmed_cooccurrence.png b/screenshots/pubmed_cooccurrence.png new file mode 100644 index 000000000..839169531 Binary files /dev/null and b/screenshots/pubmed_cooccurrence.png differ diff --git a/screenshots/pubmed_word_freq.png b/screenshots/pubmed_word_freq.png new file mode 100644 index 000000000..20521d2b9 Binary files /dev/null and b/screenshots/pubmed_word_freq.png differ diff --git a/screenshots/wos_main_info.png b/screenshots/wos_main_info.png new file mode 100644 index 000000000..f4aa78639 Binary files /dev/null and b/screenshots/wos_main_info.png differ diff --git a/www/services/__init__.py b/www/services/__init__.py index 28584e105..95ad42c29 100644 --- a/www/services/__init__.py +++ b/www/services/__init__.py @@ -2,6 +2,7 @@ from .cocmatrix import * from .couplingmap import * from .format_functions import * +from .get_data import * from .histnetwork import * from .histplot import * from .htmldownload import * @@ -14,4 +15,5 @@ from .tabletag import * from .termextraction import * from .thematicmap import * -from .utils import * \ No newline at end of file +from .utils import * +from .validation import * \ No newline at end of file diff --git a/www/services/get_data.py b/www/services/get_data.py new file mode 100644 index 000000000..2bc9baccf --- /dev/null +++ b/www/services/get_data.py @@ -0,0 +1,190 @@ +""" +get_data.py + +Defines get_data(), the function called from app.py when the user clicks +"Start" after choosing a data source and uploading file(s): + + text = get_data(input, database, df, reset_all_analyses) + +This function was referenced in app.py but never defined anywhere in the +codebase — it is the missing Load step of the ETL pipeline. Everything it +calls already exists elsewhere in www/services/ and is reused as-is: + + - biblio_json() / process_multiple_files() (format_functions.py) + Extract + Transform: raw file -> list[dict] in the WoS-style schema. + - validate_standardized_df() / coerce_to_contract() (validation.py) + QA pass: guarantees every column matches the type contract before + any analysis function ever sees the data. + - metaTagExtraction(..., Field="SR") (metatagextraction.py) + The canonical SR / SR_FULL generator, reused rather than + reimplemented (per project requirements). + +get_data() itself only orchestrates these three steps and reports the +outcome back to the UI — it contains no source-specific parsing logic. +""" + +from .utils import * +from .format_functions import biblio_json, process_multiple_files +from .metatagextraction import metaTagExtraction +from .validation import validate_standardized_df, coerce_to_contract, LIST_COLUMNS + + +# Maps the "Database:" select_input key (see app.py `select_db`) to the +# `source` argument expected by biblio_json() / process_single_file(). +_SOURCE_KEYS = {"wos", "scopus", "dimensions", "lens", "pubmed", "cochrane"} + + +def get_data(input, database, df, reset_all_analyses): + """ + Load, standardize, validate, and store the user's bibliographic dataset. + + Args: + input: The Shiny `input` object (gives access to input.Dataset(), + input.database(), input.author(), input.select()). + database: The human-readable database name returned by + get_database(input), used only for status messages. + df: The reactive.Value(None) holding the app's working DataFrame. + On success, this function calls df.set(...) with the final, + validated DataFrame. + reset_all_analyses: Callback that clears cached analysis results + whenever a new dataset is loaded. + + Returns: + A ui element (via ui.markdown / ui.HTML) describing the outcome, + which is displayed inline by the @render.express caller in app.py. + """ + mode = input.select() # "1A" raw import, "1B" load bibliometrix file, "1C" sample + + try: + if mode == "1A": + result_df = _load_raw_files(input) + elif mode == "1B": + result_df = _load_bibliometrix_file(input) + else: + # "1C" (sample dataset) is already handled directly in app.py's + # `mostra()` via pd.read_excel("sources/samples/sample.xlsx"), + # so get_data() should not normally be reached in that case. + return ui.markdown( + "ΓÜá∩╕Å No file selected. Please choose an import option and upload a file." + ) + except Exception as e: + return ui.markdown(f"Γ¥î **Error loading data:** {str(e)}") + + if result_df is None or result_df.empty: + return ui.markdown( + "Γ¥î No valid records were found in the uploaded file(s). " + "Please check the file format matches the selected database." + ) + + # --- Validate + auto-fix minor type issues ----------------------------- + report = validate_standardized_df(result_df, strict=False) + if not report["is_valid"]: + result_df = coerce_to_contract(result_df) + report = validate_standardized_df(result_df, strict=False) + + if not report["is_valid"]: + # Still invalid after coercion -> something structural is wrong + # (e.g. a required column is missing entirely). Surface this + # clearly instead of silently handing bad data to the analyses. + missing = ", ".join(report["missing_columns"]) or "none" + return ui.markdown( + f"Γ¥î **Validation failed after standardization.**\n\n" + f"- Missing columns: {missing}\n" + f"- Rows: {report['row_count']}\n\n" + "Please check the export/source file and try again." + ) + + # --- Generate SR / SR_FULL using the existing, canonical function ------ + # metaTagExtraction expects a reactive.Value wrapping the DataFrame and + # returns the same wrapper with M.set(...) already applied internally, + # so we mimic that contract with a throwaway reactive value. + sr_holder = reactive.Value(result_df) + sr_holder = metaTagExtraction(sr_holder, Field="SR") + result_df = sr_holder.get() + + # --- Store the final DataFrame and reset any cached analyses ----------- + df.set(result_df) + reset_all_analyses() + + return ui.markdown( + f"Γ£à **{database}** dataset loaded successfully: " + f"{len(result_df)} documents, {len(result_df.columns)} fields." + ) + + +def _load_raw_files(input) -> pd.DataFrame: + """ + Handle mode "1A": one or more raw exports (WoS/Scopus/Dimensions/Lens/ + PubMed/Cochrane, in .txt/.ciw/.bib/.csv/.xlsx/.zip) uploaded via + input.Dataset(). Reuses process_multiple_files() for parsing + + per-source column formatting (Extract + Transform), then converts the + resulting JSON into a DataFrame. + """ + files = input.Dataset() + if not files: + raise ValueError("No file(s) uploaded.") + + source = input.database() + if source not in _SOURCE_KEYS: + raise ValueError(f"Unrecognized database selection: {source!r}") + + author_format = input.author() # "surname" or "fullname" + + json_data = process_multiple_files(files, source, author_format) + records = json.loads(json_data) + + result_df = pd.DataFrame(records) + + # process_single_file() (format_functions.py) intentionally *deletes* + # whichever of AU/AF the user did not select as their preferred author + # name format, rather than leaving it as an empty column. That violates + # the standardized-schema contract, which requires every glossary + # column to exist (empty if unavailable) rather than being absent. + # Restore it here as an empty list per row, non-destructively. + for col in ("AU", "AF"): + if col not in result_df.columns: + result_df[col] = [[] for _ in range(len(result_df))] + + return result_df + + +def _load_bibliometrix_file(input) -> pd.DataFrame: + """ + Handle mode "1B": load a dataset previously exported from Bibliometrix + itself (.xlsx), i.e. data that is already in the standardized WoS-style + schema. Multi-value columns are stored as ';'-separated strings in the + exported Excel file (see the assignment's rule for flat-file output), + so they are re-expanded into Python lists here to satisfy the same + type contract used for freshly-parsed data. + """ + files = input.Dataset() + if not files: + raise ValueError("No file uploaded.") + + frames = [] + for file_info in files: + path = file_info["datapath"] + frame = pd.read_excel(path) + frames.append(frame) + + result_df = pd.concat(frames, ignore_index=True) if len(frames) > 1 else frames[0] + + for col in LIST_COLUMNS: + if col not in result_df.columns: + continue + result_df[col] = result_df[col].apply(_split_semicolon_list) + + return result_df + + +def _split_semicolon_list(value): + """Expand a ';'-joined string back into list[str]; pass through if already a list.""" + if isinstance(value, list): + return value + if value is None or (not isinstance(value, str) and pd.isna(value)): + return [] + if isinstance(value, str): + if not value.strip(): + return [] + return [part.strip() for part in value.split(";") if part.strip()] + return [str(value)] diff --git a/www/services/validation.py b/www/services/validation.py new file mode 100644 index 000000000..8cfc0dc06 --- /dev/null +++ b/www/services/validation.py @@ -0,0 +1,229 @@ +""" +validation.py + +Validates a standardized bibliometric DataFrame before it is handed off to +Bibliometrix-Python's analysis functions (biblionetwork, thematicmap, +histnetwork, etc.). + +This module is intentionally decoupled from the ETL logic in +format_functions.py / parsers.py: it only checks the *shape* of the data, +never re-derives or overwrites values. That separation means the same +validator can be reused for every source (WoS, Scopus, Dimensions, PubMed, +Cochrane, Lens) without knowing anything about where the DataFrame came from. + +Column type contract (must hold after standardization): + - Multi-value fields (lists in the original WoS schema, e.g. AU, DE, CR, + C1, ID, EM, FU, OA, OI, SC) -> must be Python list[str] in every row, + never a bare string, NaN, or None. Missing data -> empty list []. + - Single-value fields (e.g. TI, SO, PY, DI, VL, IS, BP, EP, SN, LA, DT, + PU, PMID, UT, SR, JI, RP, FX) -> must be a Python str in every row, + never NaN or None. Missing data -> empty string "". + - TC (Times Cited) -> must be an int (or numpy integer) in every row. + Missing data -> 0. +""" + +import pandas as pd +import numpy as np + + +# --------------------------------------------------------------------------- +# Schema definition +# --------------------------------------------------------------------------- + +# All columns that MUST exist in a valid standardized DataFrame. +# Sourced from the `columns` list already defined in www/services/utils.py, +# plus DB (added by process_single_file) and SR_FULL (added by SR() in +# metatagextraction.py). +REQUIRED_COLUMNS = [ + 'AB', 'AF', 'AU', 'AU1_UN', 'AU_UN', 'BP', 'C1', 'CR', 'DB', 'DE', + 'DI', 'DT', 'EM', 'EP', 'FU', 'FX', 'ID', 'IS', 'JI', 'LA', 'OA', + 'OI', 'PMID', 'PU', 'PY', 'RP', 'SC', 'SN', 'SO', 'SR', 'TC', 'TI', + 'UT', 'VL', +] + +# Columns whose value in every row must be a list[str]. +LIST_COLUMNS = [ + 'AF', 'AU', 'AU_UN', 'C1', 'CR', 'DE', 'EM', 'FU', 'ID', 'OA', 'OI', 'SC', +] + +# Columns whose value in every row must be a plain str. +STRING_COLUMNS = [ + 'AB', 'AU1_UN', 'BP', 'DB', 'DI', 'DT', 'EP', 'FX', 'IS', 'JI', 'LA', + 'PMID', 'PU', 'PY', 'RP', 'SN', 'SO', 'SR', 'TI', 'UT', 'VL', +] + +# Columns whose value in every row must be an int. +INT_COLUMNS = ['TC'] + + +class ValidationError(Exception): + """Raised when a standardized DataFrame fails a hard validation check.""" + pass + + +def validate_standardized_df(df: pd.DataFrame, strict: bool = True) -> dict: + """ + Validate a standardized bibliometric DataFrame against the WoS-style + column/type contract used throughout Bibliometrix-Python. + + Args: + df: The DataFrame produced by the ETL pipeline (post-format_xx_column, + pre- or post-metaTagExtraction). + strict: If True, raise ValidationError on the first hard failure + (missing required column). If False, collect all issues and + return them without raising, so the caller can decide what to + do (e.g. show a warning in the UI instead of blocking upload). + + Returns: + A report dict: + { + "is_valid": bool, + "missing_columns": [...], + "type_errors": {column: [row_indices]}, + "null_errors": {column: [row_indices]}, + "row_count": int, + "column_count": int, + } + + Raises: + ValidationError: if strict=True and required columns are missing. + """ + report = { + "is_valid": True, + "missing_columns": [], + "type_errors": {}, + "null_errors": {}, + "row_count": len(df), + "column_count": len(df.columns), + } + + # --- 1. Required columns present ------------------------------------- + missing = [c for c in REQUIRED_COLUMNS if c not in df.columns] + if missing: + report["missing_columns"] = missing + report["is_valid"] = False + if strict: + raise ValidationError( + f"Missing required column(s): {', '.join(missing)}" + ) + # If columns are missing entirely, skip type checks on them — + # there is nothing to check per-row. + + # --- 2. List-type columns: every value must be list[str] -------------- + for col in LIST_COLUMNS: + if col not in df.columns: + continue + bad_rows = df.index[df[col].apply(lambda v: not _is_valid_list(v))].tolist() + if bad_rows: + report["type_errors"][col] = bad_rows + report["is_valid"] = False + + # --- 3. String-type columns: every value must be str (never NaN/None) - + for col in STRING_COLUMNS: + if col not in df.columns: + continue + bad_rows = df.index[df[col].apply(lambda v: not _is_valid_string(v))].tolist() + if bad_rows: + report["null_errors"][col] = bad_rows + report["is_valid"] = False + + # --- 4. Int-type columns ----------------------------------------------- + for col in INT_COLUMNS: + if col not in df.columns: + continue + bad_rows = df.index[df[col].apply(lambda v: not _is_valid_int(v))].tolist() + if bad_rows: + report["type_errors"][col] = bad_rows + report["is_valid"] = False + + return report + + +def _is_valid_list(value) -> bool: + """A valid multi-value field: a Python list where every element is a str.""" + if not isinstance(value, list): + return False + return all(isinstance(item, str) for item in value) + + +def _is_valid_string(value) -> bool: + """A valid single-value field: a Python str (empty string is fine, NaN/None are not).""" + if not isinstance(value, str): + return False + return True + + +def _is_valid_int(value) -> bool: + """A valid count field: a Python or numpy integer (not float/NaN/str).""" + if isinstance(value, bool): # bool is a subclass of int — exclude it + return False + return isinstance(value, (int, np.integer)) + + +def coerce_to_contract(df: pd.DataFrame) -> pd.DataFrame: + """ + Best-effort auto-fix pass: coerce common "almost right" values into the + contract defined above, rather than failing validation outright. + + This does NOT try to recover lost information — it only fixes + representation issues (NaN -> "", NaN -> [], non-list -> [value], etc.). + Run validate_standardized_df() again after this to confirm success. + + Args: + df: The DataFrame to coerce, modified in place and returned. + + Returns: + The same DataFrame, with columns coerced to the expected types. + """ + for col in LIST_COLUMNS: + if col not in df.columns: + continue + df[col] = df[col].apply(_coerce_list_value) + + for col in STRING_COLUMNS: + if col not in df.columns: + continue + df[col] = df[col].apply(_coerce_string_value) + + for col in INT_COLUMNS: + if col not in df.columns: + continue + df[col] = df[col].apply(_coerce_int_value) + + return df + + +def _coerce_list_value(value): + if isinstance(value, list): + return [str(item) for item in value if pd.notna(item)] + if value is None or (not isinstance(value, list) and pd.isna(value)): + return [] + if isinstance(value, str): + # A stray string where a list was expected — treat as single element, + # unless it's empty/whitespace, which becomes an empty list. + return [value] if value.strip() else [] + return [str(value)] + + +def _coerce_string_value(value): + if isinstance(value, str): + return value + if value is None or (not isinstance(value, (list, dict)) and pd.isna(value)): + return "" + if isinstance(value, list): + # A stray list where a scalar was expected — join it. + return "; ".join(str(item) for item in value) + return str(value) + + +def _coerce_int_value(value): + if isinstance(value, bool): + return 0 + if isinstance(value, (int, np.integer)): + return int(value) + try: + if pd.isna(value): + return 0 + return int(float(value)) + except (TypeError, ValueError): + return 0