Pull dlt verified-sources on 2026-07-15

omniload/source/airtable/adapter.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2025 ScaleVector
+# Copyright 2022-2026 ScaleVector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
 from dlt.sources import DltResource
 
 
-@dlt.source(max_table_nesting=1)
+@dlt.source
 def airtable_source(
     base_id: str = dlt.config.value,
     table_names: Optional[List[str]] = dlt.config.value,
@@ -64,13 +64,12 @@ def airtable_resource(
             It starts with "app". See https://support.airtable.com/docs/finding-airtable-ids
         table (Dict[str, Any]): Metadata about an airtable, does not contain the actual records
     """
-
     primary_key_id = table["primaryFieldId"]
     primary_key_field = [
         field for field in table["fields"] if field["id"] == primary_key_id
     ][0]
     table_name: str = table["name"]
-    primary_key: List[str] = [f"fields__{primary_key_field['name']}".lower()]
+    primary_key: List[str] = [primary_key_field["name"]]
     air_table = api.table(base_id, table["id"])
 
     # Table.iterate() supports rich customization options, such as chunk size, fields, cell format, timezone, locale, and view

omniload/source/asana/adapter.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2025 ScaleVector
+# Copyright 2022-2026 ScaleVector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -72,7 +72,7 @@ def workspaces(
     Yields:
         dict: The workspace data.
     """
-    yield from get_client(access_token).workspaces.find_all(opt_fields=",".join(fields))  # ty: ignore[unresolved-attribute]
+    yield from get_client(access_token).workspaces.find_all(opt_fields=",".join(fields))
 
 
 @dlt.transformer(
@@ -95,7 +95,7 @@ def projects(
         list[dict]: The project data for the given workspace.
     """
     return list(
-        get_client(access_token).projects.find_all(  # ty: ignore[unresolved-attribute]
+        get_client(access_token).projects.find_all(
             workspace=workspace["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),
@@ -125,7 +125,7 @@ def sections(
     return [
         section
         for project in project_array
-        for section in get_client(access_token).sections.get_sections_for_project(  # ty: ignore[unresolved-attribute]
+        for section in get_client(access_token).sections.get_sections_for_project(
             project_gid=project["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),
@@ -151,7 +151,7 @@ def tags(
     """
     return [
         tag
-        for tag in get_client(access_token).tags.find_all(  # ty: ignore[unresolved-attribute]
+        for tag in get_client(access_token).tags.find_all(
             workspace=workspace["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),
@@ -164,10 +164,7 @@ def tasks(
     project_array: t.List[TDataItem],
     access_token: str = dlt.secrets.value,
     modified_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-        "modified_at",
-        initial_value=DEFAULT_START_DATE,
-        range_end="closed",
-        range_start="closed",
+        "modified_at", initial_value=DEFAULT_START_DATE
     ),
     fields: Iterable[str] = TASK_FIELDS,
 ) -> Iterable[TDataItem]:
@@ -185,7 +182,7 @@ def tasks(
     yield from (
         task
         for project in project_array
-        for task in get_client(access_token).tasks.find_all(  # ty: ignore[unresolved-attribute]
+        for task in get_client(access_token).tasks.find_all(
             project=project["gid"],
             timeout=REQUEST_TIMEOUT,
             modified_since=modified_at.start_value,
@@ -196,7 +193,7 @@ def tasks(
 
 @dlt.transformer(
     data_from=tasks,
-    write_disposition="replace",
+    write_disposition="append",
 )
 @dlt.defer
 def stories(
@@ -215,7 +212,7 @@ def stories(
     """
     return [
         story
-        for story in get_client(access_token).stories.get_stories_for_task(  # ty: ignore[unresolved-attribute]
+        for story in get_client(access_token).stories.get_stories_for_task(
             task_gid=task["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),
@@ -244,7 +241,7 @@ def teams(
     """
     return [
         team
-        for team in get_client(access_token).teams.find_by_organization(  # ty: ignore[unresolved-attribute]
+        for team in get_client(access_token).teams.find_by_organization(
             organization=workspace["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),
@@ -273,7 +270,7 @@ def users(
     """
     return [
         user
-        for user in get_client(access_token).users.find_all(  # ty: ignore[unresolved-attribute]
+        for user in get_client(access_token).users.find_all(
             workspace=workspace["gid"],
             timeout=REQUEST_TIMEOUT,
             opt_fields=",".join(fields),

omniload/source/asana/helpers.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2025 ScaleVector
+# Copyright 2022-2026 ScaleVector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,8 +14,7 @@
 
 """Asana source helpers"""
 
-from asana import ApiClient as AsanaClient
-from asana import Configuration
+from asana import Client as AsanaClient
 
 
 def get_client(
@@ -28,6 +27,5 @@ def get_client(
     Returns:
         AsanaClient: The Asana API client.
     """
-    configuration = Configuration()
-    configuration.access_token = access_token
-    return AsanaClient(configuration=configuration)
+    asana = AsanaClient.access_token(access_token)
+    return asana

omniload/source/asana/settings.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2025 ScaleVector
+# Copyright 2022-2026 ScaleVector
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

omniload/source/bing_webmaster/adapter.py

@@ -0,0 +1,110 @@
+# Copyright 2022-2026 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+A source loading history of organic search traffic from Bing Webmaster API
+See documentation: https://learn.microsoft.com/en-us/dotnet/api/microsoft.bing.webmaster.api.interfaces.iwebmasterapi?view=bing-webmaster-dotnet
+The API returns aggregated weekly statistics for the entire history of up to 26 weeks.
+The dates are always Fridays and during tests, the data up to the latest Friday has been available on the following Monday.
+"""
+
+import time
+from typing import Iterable, Iterator, List, Sequence
+
+import dlt
+from dlt.common import logger
+from dlt.common.typing import DictStrAny, DictStrStr
+from dlt.sources import DltResource
+
+from .helpers import get_stats_with_retry, parse_response
+
+
+@dlt.source(name="bing_webmaster")
+def source(
+    site_urls: List[str] = None, site_url_pages: Iterable[DictStrStr] = None
+) -> Sequence[DltResource]:
+    """
+    A dlt source for the Bing Webmaster api.
+    It groups resources for the APIs which return organic search traffic statistics
+    Args:
+        site_urls: List[str]: A list of site_urls, e.g, ["dlthub.com", "dlthub.de"]. Use this if you need the weekly traffic per site_url and page
+        site_url_pages: Iterable[DictStrStr]: A list of pairs of site_url and page. Use this if you need the weekly traffic per site_url, page, and query
+    Returns:
+        Sequence[DltResource]: A sequence of resources that can be selected from including page_stats and page_query_stats.
+    """
+    return (
+        page_stats(site_urls),
+        page_query_stats(site_url_pages),
+    )
+
+
+@dlt.resource(
+    write_disposition="merge",
+    merge_key=("date", "page", "site_url"),
+    primary_key=("date", "page", "site_url"),
+    table_name="bing_page_stats",
+)
+def page_stats(
+    site_urls: List[str], api_key: str = dlt.secrets.value
+) -> Iterator[Iterator[DictStrAny]]:
+    """
+    Yields detailed traffic statistics for top pages belonging to a site_url
+    Contains the entire available history of up to 26 weeks. Thus, we recommend to use write_disposition="merge"
+    API documentation:
+    https://learn.microsoft.com/en-us/dotnet/api/microsoft.bing.webmaster.api.interfaces.iwebmasterapi.getpagestats
+    Args:
+        site_urls (List[str]): List of site_urls to retrieve statistics for.
+    Yields:
+        Iterator[Dict[str, Any]]: An iterator over list of organic traffic statistics.
+    """
+    api_path = "GetPageStats"
+    for site_url in site_urls:
+        params = {"siteUrl": site_url, "apikey": api_key}
+        logger.info(f"Fetching for site_url: {site_url}")
+        response = get_stats_with_retry(api_path, params)
+        if len(response) > 0:
+            yield parse_response(response, site_url)
+
+
+@dlt.resource(
+    write_disposition="merge",
+    merge_key=("date", "page", "site_url", "query"),
+    primary_key=("date", "page", "site_url", "query"),
+    table_name="bing_page_query_stats",
+)
+def page_query_stats(
+    site_url_pages: Iterable[DictStrStr],
+    api_key: str = dlt.secrets.value,
+) -> Iterator[Iterator[DictStrAny]]:
+    """
+    Yields weekly statistics and queries for each pair of page and site_url.
+    Contains the entire available history of up to 26 weeks. Thus, we recommend to use write_disposition="merge"
+    API documentation:
+    https://learn.microsoft.com/en-us/dotnet/api/microsoft.bing.webmaster.api.interfaces.iwebmasterapi.getpagequerystats
+
+    Args:
+        site_url_page (Iterable[DictStrStr]): Iterable of site_url and pages to retrieve statistics for. Can be result of a SQL query, a parsed sitemap, etc.
+    Yields:
+        Iterator[Dict[str, Any]]: An iterator over list of organic traffic statistics.
+    """
+    api_path = "GetPageQueryStats"
+    for record in site_url_pages:
+        time.sleep(0.5)  # this avoids rate limit observed after dozens of requests
+        site_url = record.get("site_url")
+        page = record.get("page")
+        params = {"siteUrl": site_url, "page": page, "apikey": api_key}
+        logger.info(f"Fetching for site_url: {site_url}, page: {page}")
+        response = get_stats_with_retry(api_path, params)
+        if len(response) > 0:
+            yield parse_response(response, site_url, page)

omniload/source/bing_webmaster/helpers.py

@@ -0,0 +1,74 @@
+# Copyright 2022-2026 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Bing Webmaster source helpers"""
+
+import re
+from typing import Iterator, List
+from urllib.parse import urljoin
+
+from dlt.common import logger, pendulum
+from dlt.common.typing import DictStrAny, DictStrStr
+from dlt.sources.helpers import requests
+
+from .settings import BASE_URL, HEADERS
+
+
+def get_url_with_retry(url: str, params: DictStrStr) -> DictStrAny:
+    try:
+        r = requests.get(url, headers=HEADERS, params=params)
+        return r.json()  # type: ignore
+    except requests.HTTPError as e:
+        if e.response.status_code == 400:
+            logger.warning(
+                f"""HTTP Error {e.response.status_code}.
+                Is your API key authorized to fetch data about the domain
+                '{params.get("siteUrl")}'?"""
+            )
+        e.response.raise_for_status()
+        return e.response.json()  # type: ignore
+
+
+def get_stats_with_retry(api_path: str, params: DictStrStr) -> List[DictStrAny]:
+    url = urljoin(BASE_URL, api_path)
+    response = get_url_with_retry(url, params)
+    return response.get("d")  # type: ignore
+
+
+def parse_response(
+    response: List[DictStrAny], site_url: str, page: str = None
+) -> Iterator[DictStrAny]:
+    """
+    Adds site_url from the request to the response.
+    Otherwise, we would not know to which site_url a page and its statistics belong.
+    Further, corrects that what the API returns as 'Query' is actually the 'page'.
+    """
+    for r in response:
+        if page is None:
+            # in GetPageStats endpoint the page is under the key "Query"
+            r.update({"page": r.get("Query")})
+            del r["Query"]
+        else:
+            r.update({"page": page})
+        r.update({"site_url": site_url, "Date": _parse_date(r)})
+        del r["__type"]
+        yield r
+
+
+def _parse_date(record: DictStrStr) -> pendulum.Date:
+    """Parses Microsoft's date format into a date. The number is a unix timestamp"""
+    match = re.findall(r"\d+", record.get("Date"))  # extract the digits
+    timestamp_in_seconds = int(match[0]) // 1000
+    d: pendulum.Date = pendulum.Date.fromtimestamp(timestamp_in_seconds)
+    return d

omniload/source/bing_webmaster/settings.py

@@ -0,0 +1,18 @@
+# Copyright 2022-2026 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Bing Webmaster source settings and constants"""
+
+BASE_URL = "https://ssl.bing.com/webmaster/api.svc/json/"
+HEADERS = {"Content-Type": "application/json", "charset": "utf-8"}