ScrapeCommand.php

<?php
/**
 * This tool will take a list of urls, such as those generated by a backlinks tool.
 * Typically this is done for a competitive product (website) by getting backlinks using 3rd part tool.
 * These backlinks are then processed by this tool (they are the urls file list) in order to
 * extract all the emails on those links. These emails can be useful for marketing purposes (or other purposes
 *
 * Reference:
 *   https://github.com/lawzava/scrape
 *   Scrape a TSV (tab seperated file) of urls to product a TSV output with emails
 *
 * Steps in the process:
 *   1) Given the backlinks file of URLs
 *   2) Filter out url domains that are irrelevant using URL filters file
 *   3) Scrape urls to extract extract emails
 *   4) Filter out email domains that are irrelevant using email filters file (example: wix.com emails)
 *   5) De-duplicate emails in list
 *   6) Output csv file that contains linked website and email extracted
 *
 * Various options exist:
 *   - Specify custom input urls filename
 *   - Specify custom input/output emails filename
 *   - Specify custom output csv filename
 *   - Specify custom url domains filter file
 *   - Specify custom email domains filter file
 *   - Run the scrape only to generate the raw email list
 *   - Run the email filter only (to re-process a scraped list)
 *   - Do not apply a url domains filter
 *   - Do not apply an email domains filter
 */



/**
 * Helper to print a line
 *
 * @param string $output
 * @return void
 */
function printLine(string $output = null) {
  print("{$output}\n");
}

class ScrapeCommand {
  protected $argc;
  protected $argv;

  const SCRAPE_COMMAND = "./bin/scrape.darwin-amd-64 -w";
  const URLS_FILENAME_DEFAULT = 'run_scrape_urls.tsv';
  const EMAILS_FILENAME_DEFAULT = 'run_scrape_emails.csv';
  const URL_FILTERS_FILENAME_DEFAULT = 'url_filters.txt';
  const EMAIL_FILTERS_FILENAME_DEFAULT = 'email_filters.txt';
  const OUTPUT_FILENAME_DEFAULT = 'run_scrape_output.csv';

  public $urlsFile;
  public $emailsFile;
  public $urlFiltersFile;
  public $emailFiltersFile;
  public $outputFile;

  public $urlsFileName = self::URLS_FILENAME_DEFAULT;
  public $emailsFileName = self::EMAILS_FILENAME_DEFAULT;
  public $urlFiltersFileName = self::URL_FILTERS_FILENAME_DEFAULT;
  public $emailFiltersFileName = self::EMAIL_FILTERS_FILENAME_DEFAULT;
  public $outputFileName = self::OUTPUT_FILENAME_DEFAULT;

  public $urlColumn = 1;
  public $isVerbose = false;
  public $isScrapeOnly = false;
  public $isEmailOnly = false;
  public $isNoUrlFilter = false;
  public $isNoEmailFilter = false;
  public $urlsFilter = [];
  public $emailsFilter = [];

  public $statsUrlsProcessed = 0;
  public $statsUrlsInvalid = 0;
  public $statsUrlsFiltered = 0;
  public $statsEmailsTotal = 0;
  public $statsEmailsUnique = 0;
  public $statsEmailsInvalid = 0;
  public $statsEmailsFiltered = 0;

  public function __construct($argc, $argv) {
    $this->argc = $argc;
    $this->argv = $argv;
  }

  /**
   * Initialize the scrape process
   *
   * @return void
   */
  public function initialize() {
    date_default_timezone_set('America/Los_Angeles');
    $this->parseCommandLine();
    $this->validateParamsAndSetup();
  }

  /**
   * Start the scrape process
   *
   * @return void
   */
  public function process() {
    $start = time();

    $this->logLine('Timezone = ' . date_default_timezone_get());
    $this->logLine('Processing starting', true);

    if (!$this->isEmailOnly) {
      $this->executeScrape();
    }

    if (!$this->isScrapeOnly) {
      $this->processEmails();
    }

    $this->cleanup();

    $this->logLine('Processing complete', true);
    $end = time();
    $total = ($end - $start) < 120 ? ($end - $start) . ' seconds' : intval(($end - $start) / 600) * 10 . ' minutes';
    $this->logLine("  Execution time: {$total}", true);

    if (!$this->isEmailOnly) {
      $this->logLine('  URLs Processed:   ' . number_format($this->statsUrlsProcessed), true);
      $this->logLine('  URLs Filtered:    ' . number_format($this->statsUrlsFiltered), true);
      $this->logLine('  URLs Invalid:     ' . number_format($this->statsUrlsInvalid), true);
    }

    if (!$this->isScrapeOnly) {
      $this->logLine('  Emails Processed: ' . number_format($this->statsEmailsTotal), true);
      $this->logLine('  Emails Unique:    ' . number_format($this->statsEmailsUnique), true);
      $this->logLine('  Emails Filtered:  ' . number_format($this->statsEmailsFiltered), true);
    }
  }

  /*
   * Helper Functions
   */

  /**
   * Loop over all the urls and run scrape for each
   *
   * @return void
   */
  protected function executeScrape() {
    $this->logLine("Starting scrape...");

    // The first line is the column names, ignore it
    if (feof($this->urlsFile)) {
      $this->logErrorLine("Empty urls file!");
      $this->cleanupAndDie();
    }
    $line = fgets($this->urlsFile); // Header line, ignore it
    if (feof($this->urlsFile)) {
      $this->logErrorLine("Invalid urls file format, only 1 line (assumed to be column headers)!");
      $this->cleanupAndDie();
    }

    // Process until the end of the file
    while (!feof($this->urlsFile)) {
      // Reset the emailList output every time through the loop
      $emailList = [];

      $line = trim(fgets($this->urlsFile));
      $columns = explode("\t", $line);

      $url = $columns[$this->urlColumn];

      // Validate that is is in fact a valid url
      $validatedUrl = filter_var($url, FILTER_VALIDATE_URL);

      if ($validatedUrl) {
        $urlComponents = parse_url($url);
        $host = $urlComponents['host'] ?? null;

        // If filtering out this URL, count it and continue
        if ($host && array_key_exists($host, $this->urlsFilter)) {
          $this->statsUrlsFiltered++;
          continue;
        }

        // Execute the scraper with this URL
        $scrapeCommand = self::SCRAPE_COMMAND;
        exec("{$scrapeCommand} {$validatedUrl}", $emailList, $resultCode);

        if ($resultCode == 127) {
          $this->logLine("  Exec command failed with resultcode 127, the scrape command is not in your path");
          $this->cleanupAndDie();
        } else if ($resultCode != 0) {
          $this->logLine("  Exec command failed with resultcode: {$resultCode}, skipping URL {$validatedUrl}", true);
          continue;
        }
        $countUrls = count($emailList);
        $this->logLine("  Processed {$validatedUrl}, found {$countUrls} emails");

        // Pump the output data into the emails csv file
        foreach ($emailList as $email) {
          fputcsv($this->emailsFile, [$validatedUrl, $email]);
        }

        // Update the stats
        $this->statsUrlsProcessed++;
      } else {
        $this->logLine("  Skipping invalid URL: $url");
        $this->statsUrlsInvalid++;
      }
    }

    // Writing to the file is done, flush to disk and set the filepointer back to the beginning
    fflush($this->emailsFile);
    rewind($this->emailsFile);

    $this->logLine('Scrape complete', true);
  }

  /**
   * Loops over all the emails doing a de-dup and filter
   *
   * @return void
   */
  protected function processEmails() {
    $this->logLine('Starting email processing...');

    $uniqueEmails = [];

    // Loop over all the records (url & email)
    // Parse the URl into just the domain and use that an the email as array indexes
    // That will create a unique list of domains & urls
    // This means that the last one in will win if there are dups
    while (!feof($this->emailsFile)) {
      $line = trim(fgets($this->emailsFile));
      $columns = explode(',', $line);

      if (count($columns) != 2) {
        // there is a problem, skip this line
        continue;
      }

      $url = $columns[0];
      $email = $columns[1];
      $urlComponents = parse_url($url);

      // Skip this one if there is a problem
      if ($urlComponents === false || !array_key_exists('host', $urlComponents) || !filter_var($email, FILTER_VALIDATE_EMAIL)) {
        $this->statsEmailsInvalid++;
        continue;
      }

      // Save the data in an array and let the keys handle de-dup process
      $host = $urlComponents['host'];
      $emailComponents = explode('@', $email);
      $emailDomain = array_pop($emailComponents);

      // Skip this one if filtering out the email domain (also check host in case one slipped through)
      if (array_key_exists($emailDomain, $this->emailsFilter) || array_key_exists($host, $this->urlsFilter)) {
        $this->statsEmailsFiltered++;
        continue;
      }

      // Finally, add this to the list of unique emails
      $uniqueEmails[$host][$email] = $emailDomain;

      $this->statsEmailsTotal++;
    }

    // Loop over all the unique items and add them to the output file
    fputcsv($this->outputFile, ['Host', 'Email', 'EmailDomain']);
    foreach ($uniqueEmails AS $uniqueHost => $emails) {
      foreach ($emails AS $uniqueEmail => $emailDomain) {
        fputcsv($this->outputFile, [$uniqueHost, $uniqueEmail, $emailDomain]);
        $this->statsEmailsUnique++;
      }
    }

    // Flush the file to disk
    fflush($this->outputFile);

    $this->logLine('Email processing complete', true);
  }

  /**
   * Helper to validate command line params
   *
   * @return void
   */
  protected function validateParamsAndSetup() {
    // Check for mutually exclusive conditions
    if ($this->isScrapeOnly && $this->isEmailOnly) {
      $this->logErrorLine('Cannot set scrapeOnly and emailOnly options at the same time');
      $this->cleanupAndDie();
    }

    // No need to filer urls
    if ($this->isNoUrlFilter) {
      $this->urlFiltersFileName = null;
    }

    // No need to filter emails
    if ($this->isNoEmailFilter) {
      $this->emailFiltersFileName = null;
    }

    // Check that necessary files exist or can be created
    if ($this->isEmailOnly) {
      // The emails file already exists or there is a problem
      $this->emailsFile = $this->openFile($this->emailsFileName);
    } else {
      // We are going to scrape, make sure the urls file exists
      $this->urlsFile = $this->openFile($this->urlsFileName);

      // Create an empty file for the raw emails output
      $this->emailsFile = $this->createFile($this->emailsFileName);
    }

    if (!$this->isScrapeOnly) {
      // Create the output file
      $this->outputFile = $this->createFile($this->outputFileName);
    }

    // Open the url filter file and fill up the filter array
    if (!$this->isNoUrlFilter && $this->urlFiltersFileName) {
      $this->urlFiltersFile = $this->openFile($this->urlFiltersFileName);

      while (!feof($this->urlFiltersFile)) {
        // Read in the line, it should just be a domain
        $filterDomain = trim(fgets($this->urlFiltersFile));

        // Add the item to the filter
        $this->urlsFilter[$filterDomain] = 1;
      }
    }

    // Open the email filter file and fill up the filter array
    if (!$this->isNoEmailFilter && $this->emailFiltersFileName) {
      $this->emailFiltersFile = $this->openFile($this->emailFiltersFileName);

      while (!feof($this->emailFiltersFile)) {
        // Read in the line, it should just be a domain
        $filterDomain = trim(fgets($this->emailFiltersFile));

        // Add the item to the filter
        $this->emailsFilter[$filterDomain] = 1;
      }
    }
  }

  /**
   * Helper to parse the command line
   *
   * @return void
   */
  protected function parseCommandLine() {
    $arg = 1;

    while ($arg < $this->argc) {
      $nextArg = $this->argv[$arg];

      // If just want help, then output usage and die
      if ($nextArg === '--help') {
        $this->displayhelp();
        $this->cleanupAndDie();
      }

      $increment = 1;

      if ($nextArg === '--verbose') {
        $this->isVerbose = true;
      } else if ($nextArg === '--scrape-only') {
        $this->isScrapeOnly = true;
      } else if ($nextArg === '--email-only') {
        $this->isEmailOnly = true;
      } else if ($nextArg === '--no-url-filter') {
        $this->isNoUrlFilter = true;
      } else if ($nextArg === '--no-email-filter') {
        $this->isNoEmailFilter = true;
      } else if ($nextArg === '--urls') {
        $this->urlsFileName = $this->argv[$arg + 1];
        $increment = 2;
      } else if ($nextArg === '--emails') {
        $this->emailsFileName = $this->argv[$arg + 1];
        $increment = 2;
      } else if ($nextArg === '--output') {
        $this->outputFileName = $this->argv[$arg + 1];
        $increment = 2;
      } else if ($nextArg === '--url-filters') {
        $this->urlFiltersFileName = $this->argv[$arg + 1];
        $increment = 2;
      } else if ($nextArg === '--email-filters') {
        $this->emailFiltersFileName = $this->argv[$arg + 1];
        $increment = 2;
      } else if ($nextArg === '--url-column') {
        $this->urlColumn = $this->argv[$arg + 1];
        $increment = 2;
      } else {
        print("\nError! Invalid param \"{$nextArg}\"\n");
        $this->displayHelp();
        $this->cleanupAndDie();
      }

      $arg += $increment;
    }
  }

  /**
   * Helper to display tool usage
   *
   * @return void
   */
  protected function displayHelp() {
    printLine();
    printLine("Usage for run_scrape:");
    printLine("  runscrape [0 or more of the below params, all are optional]");
    printLine();
    printLine("  --help                       - display the help");
    printLine("  --verbose                    - display extra logging information");
    printLine("  --scrape-only                - only do the scrape and output the emails csv file");
    printLine("  --email-only                 - only process the emails csv file and output the result");
    printLine("  --no-url-filter              - do not apply the urls filter");
    printLine("  --no-email-filter            - do not apply the email domains filter");
    printLine();
    printLine("  --urls <filepath>            - urls to process in tsv format, defaults to {$this->urlsFileName}");
    printLine("  --emails <filepath>          - raw email list to process in csv format, defaults to {$this->emailsFileName}");
    printLine("  --output <filepath>          - output emails is csv format, defaults to {$this->outputFileName}");
    printLine("  --url-filters <filepath>     - url domains to filter out, defaults to {$this->urlFiltersFileName}");
    printLine("  --email-filters <filepath>   - email domains to filter out, defaults to {$this->emailFiltersFileName}");
    printLine("  --url-column <integer>       - defaults to column 1");
    printLine();
  }

  /***
   * Open a file without an error by first checking existence
   *
   * @param string $filepath
   * @return false|resource
   */
  protected function openFile($filepath) {
    $isFile = file_exists($filepath);
    if (!$isFile) {
      $this->logErrorLine("Could not open file: {$filepath}");
      $this->cleanupAndDie();
    }

    $file = fopen($filepath, 'r');
    if (!$file) {
      $this->logErrorLine("Could not open file: {$filepath}");
      $this->cleanupAndDie();
    }

    return $file;
  }

  /**
   * Create a file without an error by first checking existence
   *
   * @param $filepath
   * @return false|resource
   */
  protected function createFile($filepath) {
    $isFile = file_exists($filepath);
    if ($isFile) {
      $this->logErrorLine("File already exists: {$filepath}");
      $this->cleanupAndDie();
    }

    $file = fopen($filepath, 'x+');
    if (!$file) {
      $this->logErrorLine("Could not create file: {$filepath}");
      $this->cleanupAndDie();
    }

    return $file;
  }


  /**
   * Cleanup and die
   *
   * @return void
   */
  protected function cleanupAndDie() {
    $this->cleanup();
    die();
  }

  /**
   * Cleanup any necessary items (such as open files)
   *
   * @return void
   */
  protected function cleanup() {
    if ($this->outputFile) {
      fclose($this->outputFile);
    }

    if ($this->emailFiltersFile) {
      fclose($this->emailFiltersFile);
    }

    if ($this->emailsFile) {
      fclose($this->emailsFile);
    }

    if ($this->urlFiltersFile) {
      fclose($this->urlFiltersFile);
    }

    if ($this->urlsFile) {
      fclose($this->urlsFile);
    }
  }

  /***
   * Helper to log info if verbose is set
   *
   * @param string $text
   * @param bool $alwaysLog
   * @return void
   */
  protected function logLine($text, $alwaysLog = true) {
    if ($alwaysLog || $this->isVerbose) {
     printLine(date('h:i:s') . ': ' . $text);
    }
  }

  /***
   * Helper to log error info
   *
   * @param $text
   * @return void
   */
  protected function logErrorLine($text) {
    printLine(date('h:i:s') . ': [ERROR] - ' . $text);
  }

}