SimpleFeedContentGenerator.java

/**
 * Copyright (C) 2009 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 * Name: SimpleContentGenerator.java
 *
 * Description: Command line application to generate test feed content.
 *
 * The class uses prepared "words" file as source for random content. The class generates feed
 * with specified records and some random content with metadata.
 *
 * To run the program use the following two commands:
 * 1) compile
 *   javac SimpleFeedContentGenerator.java
 *
 * 2) execute
 *   java -DwordsFilePath=<ABSOLUTE_PATH_TO_WORDS_FILE> SimpleFeedContentGenerator
 *
 * "-DwordsFilePath" parameter is mandatory.
 *
 * Other parameters:
 * -DrecordsCount           - Default: 30. How many records to generate.
 * -DmetadataFieldsCount    - Default: 3. How many metadata fields to generate for each record.
 * -DmetaContentWordsCount  - Default: 3. How many words to generate in each metadata field.
 * -DbodyContentWordsCount  - Default: 0. How many words to generate in each record content body.
 * -DfeedSource             - Default: 'content'. Feed source name. Also used in recordUrl and
 *   body title generation.
 * -DrecordUrl              - Default: 'http://site.example.local/'. Record URL start path.
 * -DfeedFilePath           - Default: './feed.xml'. Distanation feed file.
 *
 * Example with all parameters:
 * java -DwordsFilePath=/usr/local/google/home/afedorenchik/downloads/content_generators/words 
 *   -DrecordsCount=30000 -DmetadataFieldsCount=40 -DmetaContentWordsCount=50 
 *   -DbodyContentWordsCount=200 -DfeedSource=articles -DrecordUrl="http://mysite.local" 
 *   -DfeedFilePath=/home/user/feeds/articles.xml SimpleFeedContentGenerator 
 *
 * This code is not supported by Google
 *
 */

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Random;

/**
 * Generates basic feed file for performance test. (generated by afedorenchik)
 */
public class SimpleFeedContentGenerator {
  private static Random randomGenerator = new Random();
  private static List<String> words;

  private static final int RECORDS_GENERATE_LIMIT_BEFORE_WRITE = 1000;

  private static void writeToFile(String s, String path) {
    try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(path, true)))) {
      out.println(s);
    } catch (IOException e) {
      e.printStackTrace();
    }
  }

  private static List<String> readWordsFile(String path) {
    Path filePath = new File(path).toPath();
    Charset charset = Charset.defaultCharset();
    List<String> stringList = null;

    try {
      stringList = Files.readAllLines(filePath, charset);
    } catch (IOException e) {
      e.printStackTrace();
    }
    return stringList;
  }

  private static String generateRandomTextData(int wordsCount) {
    StringBuilder sb = new StringBuilder();
    int i = 0;
    while (i < wordsCount) {
      sb.append(words.get(randomGenerator.nextInt(words.size())));
      sb.append(" ");
      i++;
    }
    return sb.toString();
  }

  public static void main(String[] args) {
    String wordsFilePath = null;

    int recordsCount = 30;
    int metadataFieldsCount = 3;
    int metaContentWordsCount = 3;
    int bodyContentWordsCount = 0;

    String feedSource = "sample";
    String recordUrl = "http://site.example.local/";

    String feedFilePath = "./feed.xml";

    if ((System.getProperty("recordsCount")) != null) {
      recordsCount = Integer.parseInt(System.getProperty("recordsCount"));
    }
    if (System.getProperty("metadataFieldsCount") != null) {
      metadataFieldsCount = Integer.parseInt(System.getProperty("metadataFieldsCount"));
    }
    if (System.getProperty("metaContentWordsCount") != null) {
      metaContentWordsCount = Integer.parseInt(System.getProperty("metaContentWordsCount"));
    }
    if (System.getProperty("bodyContentWordsCount") != null) {
      bodyContentWordsCount = Integer.parseInt(System.getProperty("bodyContentWordsCount"));
    }
    if (System.getProperty("feedFilePath") != null) {
      feedFilePath = System.getProperty("feedFilePath");
    }
    if (System.getProperty("feedSource") != null) {
      feedSource = System.getProperty("feedSource");
    }
    if (System.getProperty("recordUrl") != null) {
      recordUrl = System.getProperty("recordUrl");
      if (recordUrl.charAt(recordUrl.length() - 1) != '/') {
        recordUrl += "/";
      }
    }
    if (System.getProperty("wordsFilePath") != null) {
      wordsFilePath = System.getProperty("wordsFilePath");
    } else {
      System.out.println("Please use '-DwordsFilePath=<PATH_TO_WORDS_FILE>' option.");
      System.exit(1);
    }

    words = readWordsFile(wordsFilePath);

    // Check if feed file exist and delete it
    File feedFile = new File(feedFilePath);
    if (feedFile.exists())
      feedFile.delete();

    StringBuilder sb = new StringBuilder();
    sb.append("<?xml version='1.0' encoding='UTF8'?>\n");
    sb.append("<!DOCTYPE gsafeed PUBLIC '-//Google//DTD GSA Feeds//EN' ''>\n");
    sb.append("<gsafeed>\n");
    sb.append("<header>\n");
    sb.append("\t<datasource>");
    sb.append(feedSource);
    sb.append("</datasource>\n");
    sb.append("\t<feedtype>full</feedtype>\n");
    sb.append("</header>\n");
    sb.append("<group>\n");

    // First loop for records
    for (int i = 1; i < recordsCount; i++) {
      sb.append("\t<record url='");
      sb.append(recordUrl);
      sb.append(feedSource);
      sb.append("/page");
      sb.append(i);
      sb.append("' mimetype='text/html'>\n");
      sb.append("\t\t<metadata>\n");

      // Generate metadata name and content
      for (int j = 0; j < metadataFieldsCount; j++) {
        sb.append("\t\t\t<meta name=\"");
        sb.append(words.get(randomGenerator.nextInt(words.size())));
        sb.append("\" content=\"");
        sb.append(generateRandomTextData(metaContentWordsCount).trim());
        sb.append("\"/>\n");
      }
      sb.append("\t\t</metadata>\n");
      sb.append("\t\t<content>\n");
      sb.append("\t\t\t<![CDATA[<html><head><title>");
      sb.append("Namaste - ");
      sb.append(feedSource);
      sb.append(" - Page ");
      sb.append(i);
      sb.append("</title></head>");
      sb.append("<body><p>This is hello");
      sb.append(i);
      sb.append(". Simple content for crawler.</p><div>Lorem ipsum dolor sit amet, consectetuer ");
      sb.append("adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis ");
      sb.append("et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ");
      sb.append("natoque penatibus ultricies nec, pellentesque eu, pretium quis, sem. ");
      sb.append("Nulla consequat massa quis enim. Donec pede justo, fringilla vel, ");
      sb.append("aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, ");
      sb.append("imperdiet a, venenatis vitae, justo. Nullam dictum felis eu ");
      sb.append("pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum ");
      sb.append("semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, ");
      sb.append("porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, ");
      sb.append(" dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut ");
      sb.append("metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies ");
      sb.append("nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam ");
      sb.append("rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper ");
      sb.append("libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, ");
      sb.append("luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante ");
      sb.append("tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. ");
      sb.append("Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt.</div>");
      sb.append("<p>");
      sb.append(generateRandomTextData(bodyContentWordsCount).trim());
      sb.append("</p>");
      sb.append("</body></html>]]>\n");
      sb.append("\t\t</content>\n");
      sb.append("\t</record>\n");

      // Need to write each 1000 records to not over heap
      if ((i % RECORDS_GENERATE_LIMIT_BEFORE_WRITE) == 0) {
        String s = sb.toString();
        writeToFile(s, feedFilePath);
        // Flush StringBuffer
        sb.setLength(0);
      }
    }
    sb.append("</group>\n");
    sb.append("</gsafeed>\n");
    String s = sb.toString();
    writeToFile(s, feedFilePath);
  }
}