From 7eb023891cbc54d364ed48465da9fec73d4f7a26 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 22:33:55 +0700 Subject: [PATCH 01/11] fix: fix version github actions --- .github/workflows/run_scraper.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run_scraper.yml b/.github/workflows/run_scraper.yml index ec8a8d5..5223855 100644 --- a/.github/workflows/run_scraper.yml +++ b/.github/workflows/run_scraper.yml @@ -6,6 +6,9 @@ on: - cron: '0 */12 * * *' workflow_dispatch: # Cho phép bạn bấm nút "Run" thủ công trên GitHub +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: runs-on: ubuntu-latest From 107662076dbe3425c500d4d911f4771742a58cd3 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 22:34:51 +0700 Subject: [PATCH 02/11] fix: fix version github actions --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index caf01c7..7e868a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,6 @@ MarkupSafe==3.0.3 mashumaro==3.14 more-itertools==10.8.0 msgpack==1.1.2 -networkx==3.6.1 orderly-set==5.5.0 packaging==26.0 parsedatetime==2.6 From ae4ec3ef638104db16985c3a41e11bc58d6f2be7 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 22:43:47 +0700 Subject: [PATCH 03/11] fix: fix version github actions --- crawl/TopDevJob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index fd354d9..2c1260a 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -85,7 +85,7 @@ async def crawl_all_pages(self, today = False): # Nếu KHÔNG chứa 'opacity-0' thì mới là nút bấm được if "opacity-0" not in class_attr: print("➡️ Nút Next đang sẵn sàng, bấm để sang trang tiếp...") - await next_button.click() + await next_button.click(force=True) # Dùng force để đảm bảo click dù có phần tử nào đó chồng lên current_page += 1 # Đợi dữ liệu mới nạp xong await self.page.wait_for_load_state("networkidle") From 3a7014021394d6a764afa2a3deab5e9321af33e4 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 22:48:59 +0700 Subject: [PATCH 04/11] fix: fix version github actions --- crawl/TopDevJob.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 2c1260a..9a8eae2 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -78,13 +78,13 @@ async def crawl_all_pages(self, today = False): # Kiểm tra nếu nút Next còn tồn tại và có thể click được # (Nút Next cuối cùng thường có class opacity-0 hoặc hidden) - if await next_button.count() > 0: + if await next_button.count() > 0 and await next_button.is_visible() and await next_button.is_enabled(): # Lấy class để kiểm tra xem có bị ẩn (trang cuối) không class_attr = await next_button.get_attribute("class") # Nếu KHÔNG chứa 'opacity-0' thì mới là nút bấm được if "opacity-0" not in class_attr: - print("➡️ Nút Next đang sẵn sàng, bấm để sang trang tiếp...") + print("➡️ Next button is visible and enabled. Clicking to go to the next page...") await next_button.click(force=True) # Dùng force để đảm bảo click dù có phần tử nào đó chồng lên current_page += 1 # Đợi dữ liệu mới nạp xong @@ -92,7 +92,7 @@ async def crawl_all_pages(self, today = False): # Đợi thêm 1s để chắc chắn các card cũ đã bị thay thế (tránh cào trùng) await self.page.wait_for_timeout(1000) else: - print("🏁 Đã thấy nút Next nhưng nó bị ẩn (Trang cuối rồi).") + print("🏁 Last page reached.") return all_jobs async def crawl_today(self): From 11f3c523a07dd15cb042b6c51928bd742e26be8e Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 22:58:58 +0700 Subject: [PATCH 05/11] fix: fix version github actions --- crawl/TopDevJob.py | 2 +- main.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 9a8eae2..36c52f2 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -81,7 +81,7 @@ async def crawl_all_pages(self, today = False): if await next_button.count() > 0 and await next_button.is_visible() and await next_button.is_enabled(): # Lấy class để kiểm tra xem có bị ẩn (trang cuối) không class_attr = await next_button.get_attribute("class") - + # Nếu KHÔNG chứa 'opacity-0' thì mới là nút bấm được if "opacity-0" not in class_attr: print("➡️ Next button is visible and enabled. Clicking to go to the next page...") diff --git a/main.py b/main.py index b8a95c6..c1c5804 100644 --- a/main.py +++ b/main.py @@ -24,6 +24,7 @@ async def test_scraper(): # 4. Gọi hàm crawl jobs = await scraper.crawl_all_pages(today=True) # Nếu bạn chỉ muốn crawl hôm nay thì truyền today=True) print(f"✅ Crawled {len(jobs)} jobs from TopDev.vn") + scraper.print_jobs(jobs) # In ra console để kiểm tra # 5. In kết quả ra màn hình để kiểm tra for job in jobs: scraper.send_to_discord(job) From d1ceacff2dc813484f6768326c82a6b41735bdc4 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 23:09:51 +0700 Subject: [PATCH 06/11] fix: fix version github actions --- crawl/TopDevJob.py | 1 + 1 file changed, 1 insertion(+) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 36c52f2..2b45d22 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -104,6 +104,7 @@ async def crawl_today(self): # print(f"📄 Processing card {i+1}/{len(cards)}...") card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết + print(job_data.title) if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1 and job_data.posted_date.find("hours") != -1): # Hồ Chí Minh Hà Nội jobs.append(job_data) return jobs From 1ef0f5dc3cd7e55750d499af4e5aefe99f069a70 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 23:30:42 +0700 Subject: [PATCH 07/11] fix: fix version github actions --- crawl/TopDevJob.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 2b45d22..410f9a5 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -79,22 +79,23 @@ async def crawl_all_pages(self, today = False): # Kiểm tra nếu nút Next còn tồn tại và có thể click được # (Nút Next cuối cùng thường có class opacity-0 hoặc hidden) if await next_button.count() > 0 and await next_button.is_visible() and await next_button.is_enabled(): + # Lấy class để kiểm tra xem có bị ẩn (trang cuối) không class_attr = await next_button.get_attribute("class") - - # Nếu KHÔNG chứa 'opacity-0' thì mới là nút bấm được - if "opacity-0" not in class_attr: - print("➡️ Next button is visible and enabled. Clicking to go to the next page...") - await next_button.click(force=True) # Dùng force để đảm bảo click dù có phần tử nào đó chồng lên - current_page += 1 - # Đợi dữ liệu mới nạp xong - await self.page.wait_for_load_state("networkidle") - # Đợi thêm 1s để chắc chắn các card cũ đã bị thay thế (tránh cào trùng) - await self.page.wait_for_timeout(1000) - else: - print("🏁 Last page reached.") + if "pointer-events-none opacity-0" in class_attr: + print("🚫 Next button is present but disabled (opacity-0 or pointer-events-none). This is the last page!") return all_jobs + # Nếu KHÔNG chứa 'opacity-0' thì mới là nút bấm được + print("➡️ Next button is visible and enabled. Clicking to go to the next page...") + await next_button.click(force=True) # Dùng force để đảm bảo click dù có phần tử nào đó chồng lên + current_page += 1 + # Đợi dữ liệu mới nạp xong + await self.page.wait_for_load_state("networkidle") + # Đợi thêm 1s để chắc chắn các card cũ đã bị thay thế (tránh cào trùng) + await self.page.wait_for_timeout(1000) + + async def crawl_today(self): jobs = [] container = self.page.locator("div.flex-col.gap-2").first @@ -104,7 +105,6 @@ async def crawl_today(self): # print(f"📄 Processing card {i+1}/{len(cards)}...") card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết - print(job_data.title) if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1 and job_data.posted_date.find("hours") != -1): # Hồ Chí Minh Hà Nội jobs.append(job_data) return jobs From 864a7ba6d2265d009c1564eb1ff603e26254feb5 Mon Sep 17 00:00:00 2001 From: loc Date: Mon, 13 Apr 2026 23:43:50 +0700 Subject: [PATCH 08/11] fix: fix version github actions --- crawl/TopDevJob.py | 14 +++++++------- models/Job.py | 10 +++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 410f9a5..1532dc5 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -10,7 +10,7 @@ def __init__(self, page, webhook_url): self.url = "https://topdev.vn/jobs/search" self.find_level = ["Intern", "Fresher", "Junior"] async def crawl(self): - jobs = [] + jobs = set() container = self.page.locator("div.flex-col.gap-2").first cards = await container.locator(".text-card-foreground").all() print(f"🔍 Found {len(cards)} job cards") @@ -19,7 +19,7 @@ async def crawl(self): card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1): # Hồ Chí Minh Hà Nội - jobs.append(job_data) + jobs.add(job_data) return jobs async def parse_card_detail(self, card): @@ -65,14 +65,14 @@ async def crawl_all_pages(self, today = False): roles = ["Software Developer", "Data Engineer / Scientist / Analyst", "Machine Learning / AI Engineer", "DevOps Engineer"] for role in roles: await self.page.locator('div[style*="width:600px"] button').filter(has_text=role).click() - all_jobs = [] + all_jobs = set() await self.page.get_by_role("button", name="Apply", exact=True).click() while True: print(f"🚅 Crawling page {current_page}...") if(today): - all_jobs.extend(await self.crawl_today()) + all_jobs.update(await self.crawl_today()) else: - all_jobs.extend(await self.crawl()) + all_jobs.update(await self.crawl()) next_button = self.page.get_by_label("Go to next page") @@ -97,7 +97,7 @@ async def crawl_all_pages(self, today = False): async def crawl_today(self): - jobs = [] + jobs = set() container = self.page.locator("div.flex-col.gap-2").first cards = await container.locator(".text-card-foreground").all() print(f"🔍 Found {len(cards)} job cards") @@ -106,7 +106,7 @@ async def crawl_today(self): card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1 and job_data.posted_date.find("hours") != -1): # Hồ Chí Minh Hà Nội - jobs.append(job_data) + jobs.add(job_data) return jobs def send_to_discord(self, job_data): diff --git a/models/Job.py b/models/Job.py index 55dc3b7..19cf965 100644 --- a/models/Job.py +++ b/models/Job.py @@ -21,4 +21,12 @@ def to_discord_embed(self): "color": 5814783, # Màu xanh dương "footer": {"text": f"Nguồn: {self.posted_date}"}, "image": {"url": self.image} - } \ No newline at end of file + } + def __eq__(self, other): + if not isinstance(other, Job): + return False + return self.link == other.link + + # Hàm tạo mã hash dựa trên link + def __hash__(self): + return hash(self.link) \ No newline at end of file From 3991ece548806a94a12c9914b931d273b689d90b Mon Sep 17 00:00:00 2001 From: loc Date: Tue, 14 Apr 2026 00:01:38 +0700 Subject: [PATCH 09/11] fix: fix version github actions --- crawl/TopDevJob.py | 19 ++++++++++++------- main.py | 1 + models/Job.py | 9 +-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 1532dc5..7f7ff54 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -9,8 +9,9 @@ def __init__(self, page, webhook_url): super().__init__(page = page, webhook_url = webhook_url) self.url = "https://topdev.vn/jobs/search" self.find_level = ["Intern", "Fresher", "Junior"] + self.scraped_links = set() # Dùng để lưu các link đã cào được, tránh trùng lặp khi crawl nhiều trang async def crawl(self): - jobs = set() + jobs = [] container = self.page.locator("div.flex-col.gap-2").first cards = await container.locator(".text-card-foreground").all() print(f"🔍 Found {len(cards)} job cards") @@ -18,8 +19,10 @@ async def crawl(self): # print(f"📄 Processing card {i+1}/{len(cards)}...") card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết - if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1): # Hồ Chí Minh Hà Nội - jobs.add(job_data) + if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1 ): # Hồ Chí Minh Hà Nội + if job_data.link not in self.scraped_links: + jobs.append(job_data) + self.scraped_links.add(job_data.link) # Ghi chú lại link này return jobs async def parse_card_detail(self, card): @@ -70,9 +73,9 @@ async def crawl_all_pages(self, today = False): while True: print(f"🚅 Crawling page {current_page}...") if(today): - all_jobs.update(await self.crawl_today()) + all_jobs.extend(await self.crawl_today()) else: - all_jobs.update(await self.crawl()) + all_jobs.extend(await self.crawl()) next_button = self.page.get_by_label("Go to next page") @@ -97,7 +100,7 @@ async def crawl_all_pages(self, today = False): async def crawl_today(self): - jobs = set() + jobs = [] container = self.page.locator("div.flex-col.gap-2").first cards = await container.locator(".text-card-foreground").all() print(f"🔍 Found {len(cards)} job cards") @@ -106,7 +109,9 @@ async def crawl_today(self): card = cards[i] job_data = await self.parse_card_detail(card) # Hàm bóc tách chi tiết đã viết if(job_data.exp in self.find_level and job_data.address.find("Hồ Chí Minh") != -1 and job_data.posted_date.find("hours") != -1): # Hồ Chí Minh Hà Nội - jobs.add(job_data) + if job_data.link not in self.scraped_links: + jobs.append(job_data) + self.scraped_links.add(job_data.link) # Ghi chú lại link này return jobs def send_to_discord(self, job_data): diff --git a/main.py b/main.py index c1c5804..2e7adf3 100644 --- a/main.py +++ b/main.py @@ -26,6 +26,7 @@ async def test_scraper(): print(f"✅ Crawled {len(jobs)} jobs from TopDev.vn") scraper.print_jobs(jobs) # In ra console để kiểm tra # 5. In kết quả ra màn hình để kiểm tra + jobs = list(set(jobs)) # Loại bỏ trùng lặp nếu có for job in jobs: scraper.send_to_discord(job) except Exception as e: diff --git a/models/Job.py b/models/Job.py index 19cf965..11d62c1 100644 --- a/models/Job.py +++ b/models/Job.py @@ -22,11 +22,4 @@ def to_discord_embed(self): "footer": {"text": f"Nguồn: {self.posted_date}"}, "image": {"url": self.image} } - def __eq__(self, other): - if not isinstance(other, Job): - return False - return self.link == other.link - - # Hàm tạo mã hash dựa trên link - def __hash__(self): - return hash(self.link) \ No newline at end of file + \ No newline at end of file From ae4e69f43b510331d8a137d542b0e1104bfca607 Mon Sep 17 00:00:00 2001 From: loc Date: Tue, 14 Apr 2026 00:04:22 +0700 Subject: [PATCH 10/11] fix: fix version github actions --- crawl/TopDevJob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl/TopDevJob.py b/crawl/TopDevJob.py index 7f7ff54..77707d5 100644 --- a/crawl/TopDevJob.py +++ b/crawl/TopDevJob.py @@ -68,7 +68,7 @@ async def crawl_all_pages(self, today = False): roles = ["Software Developer", "Data Engineer / Scientist / Analyst", "Machine Learning / AI Engineer", "DevOps Engineer"] for role in roles: await self.page.locator('div[style*="width:600px"] button').filter(has_text=role).click() - all_jobs = set() + all_jobs = [] await self.page.get_by_role("button", name="Apply", exact=True).click() while True: print(f"🚅 Crawling page {current_page}...") From 2b76cb510b92a2e5cfbb8058e8c60f369439d5e1 Mon Sep 17 00:00:00 2001 From: loc Date: Tue, 14 Apr 2026 00:10:13 +0700 Subject: [PATCH 11/11] fix: fix version github actions --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 2e7adf3..c1c5804 100644 --- a/main.py +++ b/main.py @@ -26,7 +26,6 @@ async def test_scraper(): print(f"✅ Crawled {len(jobs)} jobs from TopDev.vn") scraper.print_jobs(jobs) # In ra console để kiểm tra # 5. In kết quả ra màn hình để kiểm tra - jobs = list(set(jobs)) # Loại bỏ trùng lặp nếu có for job in jobs: scraper.send_to_discord(job) except Exception as e: