From af40f758903257058b03c79f481964ec1caffc78 Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Sun, 18 Jan 2026 22:14:38 +0800 Subject: [PATCH 1/6] Revert "corrected crawler README.md MySQL link" --- crawler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler/README.md b/crawler/README.md index fe4f211..679b064 100644 --- a/crawler/README.md +++ b/crawler/README.md @@ -61,7 +61,7 @@ npm start ## ⛏️ Built Using -- [MYSQL](https://www.mysql.com/) - Database +- [MYSQL](https://www.mongodb.com/) - Database - [NodeJs](https://nodejs.org/en/) - Server Environment - [StellarSDK](https://github.com/stellar/js-stellar-sdk) - BlockchainTool ## ✍️ Authors From c81bf0c378c8dbb9b891edd4fcefad1690fdf2db Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Mon, 19 Jan 2026 08:02:46 +0800 Subject: [PATCH 2/6] Create CONTRIBUTING.md --- crawler/CONTRIBUTING.md | 224 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 crawler/CONTRIBUTING.md diff --git a/crawler/CONTRIBUTING.md b/crawler/CONTRIBUTING.md new file mode 100644 index 0000000..22f3b8c --- /dev/null +++ b/crawler/CONTRIBUTING.md @@ -0,0 +1,224 @@ +# Contributing to Crawler + +Thank you for your interest in contributing to the Crawler project! We welcome contributions from everyone. This document provides guidelines and instructions for contributing. + +## Code of Conduct + +We are committed to providing a welcoming and inspiring community for all. Please be respectful and constructive in all interactions. Harassment, discrimination, or disruptive behavior will not be tolerated. + +## How to Contribute + +There are many ways to contribute to this project: + +- **Report bugs** by opening an issue with detailed information +- **Suggest features** with clear use cases and expected behavior +- **Improve documentation** by fixing typos or clarifying confusing sections +- **Submit code changes** by creating pull requests with meaningful improvements +- **Review pull requests** and provide constructive feedback to other contributors + +## Getting Started + +### Prerequisites + +- Python 3.8 or higher +- Git +- A MySQL database for testing (optional but recommended) +- A code editor or IDE of your choice + +### Setting Up Your Development Environment + +1. Fork the repository on GitHub +2. Clone your fork locally: + ```bash + git clone https://github.com/your-username/crawler.git + cd crawler + ``` +3. Create a virtual environment: + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` +4. Install development dependencies: + ```bash + pip install -r requirements-dev.txt + ``` +5. Create a local `.env` file for testing: + ```bash + cp .env.example .env + ``` + +## Making Changes + +### Branch Naming + +Create a descriptive branch name for your changes: +- `feature/add-proxy-support` +- `bugfix/fix-mysql-connection-timeout` +- `docs/improve-readme` +- `test/add-crawler-tests` + +```bash +git checkout -b feature/your-feature-name +``` + +### Code Style + +Follow these guidelines to maintain consistent code quality: + +- Use PEP 8 style guide for Python code +- Keep lines under 100 characters when possible +- Use meaningful variable and function names +- Add docstrings to functions and classes +- Use type hints where applicable + +Example: +```python +def fetch_url(url: str, timeout: int = 10) -> str: + """ + Fetch content from a given URL. + + Args: + url: The URL to fetch + timeout: Request timeout in seconds (default: 10) + + Returns: + The HTML content of the page + + Raises: + requests.exceptions.RequestException: If the request fails + """ + response = requests.get(url, timeout=timeout) + response.raise_for_status() + return response.text +``` + +### Testing + +Before submitting a pull request, ensure your code passes all tests: + +```bash +# Run all tests +pytest + +# Run tests with coverage +pytest --cov=crawler + +# Run specific test file +pytest tests/test_crawler.py +``` + +Write tests for new features: +```python +def test_fetch_url_success(): + """Test that fetch_url returns content for valid URLs.""" + result = fetch_url("https://example.com") + assert result is not None + assert len(result) > 0 +``` + +### Commits + +Write clear, descriptive commit messages: + +```bash +# Good +git commit -m "Add proxy support to crawler + +- Add ProxyManager class to handle proxy rotation +- Update fetch_url to accept proxy configuration +- Add tests for proxy connection handling" + +# Avoid +git commit -m "fix stuff" +git commit -m "changes" +``` + +## Submitting Changes + +### Pull Request Process + +1. Ensure all tests pass and code is formatted correctly +2. Push your branch to your fork: + ```bash + git push origin feature/your-feature-name + ``` +3. Open a pull request on GitHub with: + - A clear title describing the change + - A detailed description of what was changed and why + - Reference to any related issues (e.g., "Fixes #123") + - Screenshots or examples if applicable +4. Address review comments and make requested changes +5. Ensure the CI/CD pipeline passes +6. Once approved, your PR will be merged + +### Pull Request Template + +```markdown +## Description +Brief explanation of what this PR does. + +## Changes Made +- Change 1 +- Change 2 +- Change 3 + +## Related Issues +Fixes #123 + +## Testing +Describe how you tested these changes. + +## Checklist +- [ ] Code follows style guidelines +- [ ] Tests pass locally +- [ ] Documentation is updated +- [ ] No breaking changes (or documented in PR) +``` + +## Reporting Bugs + +When reporting bugs, please include: + +- **Description**: What you were trying to do +- **Expected behavior**: What should have happened +- **Actual behavior**: What actually happened +- **Environment**: Python version, OS, MySQL version +- **Steps to reproduce**: Clear steps to replicate the issue +- **Error message**: Full error traceback if available +- **Screenshots**: If applicable + +Example: +``` +Title: Crawler fails with timeout on large datasets + +Description: When crawling more than 10,000 pages, the crawler +consistently times out. + +Steps to reproduce: +1. Configure crawler with 15,000 pages +2. Run `python crawler.py` +3. After ~8,000 pages, connection fails + +Expected: Crawler should complete all 15,000 pages +Actual: Crawler crashes with timeout error + +Environment: Python 3.9, Ubuntu 20.04, MySQL 8.0 +``` + +## Suggesting Features + +When suggesting features, explain: + +- **Use case**: Why this feature is needed +- **Expected behavior**: How it should work +- **Alternative approaches**: Other possible implementations +- **Impact**: How it affects existing functionality + +## Documentation + +Help improve documentation by: + +- Fixing typos and grammatical errors +- Adding missing sections or examples +- Clarifying confusing explanations +- Adding inline code comments for complex logic \ No newline at end of file From bc3596690d1489f8520e06b76c342d21e77d87e6 Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Mon, 19 Jan 2026 10:08:37 +0800 Subject: [PATCH 3/6] Create LICENSE --- crawler/LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 crawler/LICENSE diff --git a/crawler/LICENSE b/crawler/LICENSE new file mode 100644 index 0000000..bc7a6e5 --- /dev/null +++ b/crawler/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Crawler Project Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file From cc6ed8f4a7c2608bb7a97f16985728877d5d86d8 Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Sun, 8 Feb 2026 22:10:51 +0800 Subject: [PATCH 4/6] Create homepage for Pi Network Added a new homepage with HTML and CSS for the Pi Network. --- website/public/homepage.png | 1142 +++++++++++++++++++++++++++++++++++ 1 file changed, 1142 insertions(+) create mode 100644 website/public/homepage.png diff --git a/website/public/homepage.png b/website/public/homepage.png new file mode 100644 index 0000000..684d2cb --- /dev/null +++ b/website/public/homepage.png @@ -0,0 +1,1142 @@ + + + + + + Pi Network - Mine Cryptocurrency on Your Phone + + + +
+ +
+ + +
+ + +
+
+

Mine Pi
on Your
Phone

+

+ Join millions of Pioneers mining cryptocurrency on mobile. + Build the world's most inclusive peer-to-peer marketplace and online experience, fueled by Pi. +

+
+ + 📱 Download Pi App + + + Learn More + +
+
+ +
+
+ π +
+
+ π +
+
+ π +
+
+ π +
+
+ 📱 +
+
+
+ 🌐 +
+
+
+ + +
+
+
0
+
Pioneers Worldwide
+
+
+
0
+
Countries
+
+
+
0
+
Daily Active Users
+
+
+
$0
+
Mining Cost
+
+
+ + +
+

How Pi Mining Works

+
+
+
1
+

Download

+

Get the Pi Network app from your app store and create your account securely

+
+
+
2
+

Verify

+

Complete identity verification to unlock full mining potential and features

+
+
+
3
+

Mine

+

Tap once daily to activate mining and earn Pi cryptocurrency effortlessly

+
+
+
4
+

Build

+

Grow your security circle and use Pi in the ecosystem marketplace

+
+
+
+ + +
+
+
📱
+

Mobile-First Mining

+

Mine cryptocurrency directly from your smartphone without draining battery or consuming data

+
+
+
🔒
+

Secure & Trusted

+

Built by Stanford PhDs with cutting-edge Stellar Consensus Protocol and advanced security

+
+
+
🌍
+

Global Community

+

Join the largest decentralized community of everyday people building the future of Web3

+
+
+
💎
+

Real Utility

+

Use Pi to buy goods, services, and experiences in the rapidly growing Pi ecosystem

+
+
+
⚡
+

Energy Efficient

+

Eco-friendly mining that doesn't require massive computational power or electricity

+
+
+
🎯
+

Fair Distribution

+

Equal opportunity for everyone to earn Pi, no matter where you are in the world

+
+
+ + +
+

Disclaimer

+

+ This presentation is for informational purposes only and contains statements about the future, including anticipated programs and features, developments, and timelines for the rollout of these programs and features. These statements are only predictions and reflect current beliefs and expectations with respect to future events; they are based on assumptions and are subject to risk, uncertainties, and change at any time. There can be no guarantee that any of the contemplated programs or features will be implemented as specified nor any assurance that actual results will not differ materially from those expressed or implied in these statements, although we believe them to be based on reasonable assumptions. All statements are valid only as of the date first presented. The statements in this presentation also may not reflect future developments due to user feedback or later events and we may not update this presentation in response. Pi Network is not a financial advisor and this is not financial advice. +

+
+ + + +
+ + + + From 6c7415b8cc7b49ad3fb94ddb0d123e18d47d6227 Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Mon, 9 Feb 2026 15:09:54 +0800 Subject: [PATCH 5/6] Rename homepage.png to homepage.html --- website/public/{homepage.png => homepage.html} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename website/public/{homepage.png => homepage.html} (100%) diff --git a/website/public/homepage.png b/website/public/homepage.html similarity index 100% rename from website/public/homepage.png rename to website/public/homepage.html From 59b0c16e5e230f5645e1830e2ddc67ef38be6e5c Mon Sep 17 00:00:00 2001 From: Marfin <248203147+arifinahmad99-cloud@users.noreply.github.com> Date: Fri, 20 Feb 2026 21:07:12 +0800 Subject: [PATCH 6/6] Update devcontainer configuration for Python environment --- .devcontainer/devcontainer.json | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..79e008b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,42 @@ +{ + "name": "ExplorePi Python Environment", + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + + "features": { + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "ms-python.black-formatter", + "ms-python.isort", + "ms-python.flake8", + "charliermarsh.ruff", + "github.copilot", + "eamodio.gitlens" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.formatting.provider": "black", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false + } + } + }, + + "forwardPorts": [8000, 5000, 3000], + + "postCreateCommand": "pip install --upgrade pip && pip install -r requirements.txt", + + "remoteUser": "vscode" +}