build.gradle

buildscript {
    repositories {
        mavenCentral()
    }
    dependencies {
        classpath 'de.thetaphi:forbiddenapis:3.10'
    }
}
plugins {
    id 'java-library'
    id 'maven-publish'
    // https://github.com/SgtSilvio/gradle-maven-central-publishing
    id 'io.github.sgtsilvio.gradle.maven-central-publishing' version '0.4.0'
    id 'io.codearte.nexus-staging' version '0.30.0'
}

apply plugin: 'de.thetaphi.forbiddenapis'
apply plugin: 'jacoco'
apply plugin: 'signing'

java {
  toolchain {
	  sourceCompatibility = JavaVersion.VERSION_21
  }
}
group = 'org.dstadler'
base {
    archivesName = 'commoncrawldownload'
}

repositories {
    mavenCentral()
}

forbiddenApis {
    suppressAnnotations = ['org.dstadler.commons.util.SuppressForbidden']
    bundledSignatures = [ 'jdk-reflection', 'commons-io-unsafe-2.19.0', 'jdk-internal' ]
    signaturesFiles += files('config/forbidden-apis/http-signatures.txt')
}
forbiddenApisMain {
    // 'jdk-unsafe', 'jdk-system-out'
    bundledSignatures += [ 'jdk-deprecated', 'jdk-internal', 'jdk-non-portable' ]
    signaturesFiles += files('config/forbidden-apis/forbidden.signatures.txt')
}

dependencies {
    implementation 'org.dstadler:commons-dost:1.4.4'
    implementation 'org.apache.httpcomponents.client5:httpclient5:5.5.1'
    implementation 'org.apache.commons:commons-lang3:3.20.0'
    implementation 'commons-io:commons-io:2.21.0'
    implementation 'commons-codec:commons-codec:1.20.0'

    implementation 'org.netpreserve.commons:webarchive-commons:1.1.11'
    implementation 'com.fasterxml.jackson.core:jackson-core:2.20.1'

    // for commoncrawl-source
    implementation 'org.jsoup:jsoup:1.15.3'

    // for @Nullable
    implementation 'com.google.code.findbugs:jsr305:3.0.2'
    testImplementation 'org.junit.jupiter:junit-jupiter-api:6.0.1'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:6.0.1'
    testRuntimeOnly 'org.junit.platform:junit-platform-launcher:6.0.1'

    testImplementation 'org.dstadler:commons-test:1.0.0.22'
}

wrapper {
    gradleVersion = '8.14.3'
}

test {
    systemProperties = System.properties as Map<String, ?>

    // enable to show standard out and standard error of the test JVM(s) on the console
    // testLogging.showStandardStreams = true

    // http://forums.gradle.org/gradle/topics/jacoco_related_failure_in_multiproject_build
    systemProperties['user.dir'] = workingDir

    useJUnitPlatform()
}

jacoco {
    toolVersion = '0.8.14'
}

jacocoTestReport {
    reports {
        xml.required = true
    }
}

tasks.register('sourcesJar', Jar) {
    archiveClassifier = 'sources'
    from sourceSets.main.allJava
}

tasks.register('javadocJar', Jar) {
    archiveClassifier = 'javadoc'
    from javadoc
}

tasks.withType(Jar).configureEach { task ->
    task.doLast {
        ant.checksum algorithm: 'md5', file: it.archiveFile.get().asFile
        ant.checksum algorithm: 'sha1', file: it.archiveFile.get().asFile
    }
}

java {
    withJavadocJar()
    withSourcesJar()
}

publishing {
    publications {
        mavenJava(MavenPublication) {
            artifactId = 'commoncrawldownload'
            from components.java

            pom {
                name = 'CommonCrawl Document Download'
                description = 'Helper tools to download data from CommonCrawl archives'
                url = 'https://github.com/centic9/CommonCrawlDocumentDownload'
                licenses {
                    license {
                        name = 'BSD 2-Clause "Simplified" License'
                        url = 'https://github.com/centic9/CommonCrawlDocumentDownload/blob/master/LICENSE.md'
                    }
                }
                developers {
                    developer {
                        id = 'centic9'
                        name = 'Dominik Stadler'
                    }
                }
                scm {
                    connection = 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
                    developerConnection = 'scm:git@github.com:centic9/CommonCrawlDocumentDownload'
                    url = 'https://github.com/centic9/CommonCrawlDocumentDownload'
                }
            }
        }
    }
    repositories {
        maven {
            // change to point to your repo, e.g. http://my.org/repo
            //url = layout.buildDirectory.dir('repo')
            url = 'https://oss.sonatype.org/service/local/staging/deploy/maven2/'
            credentials(PasswordCredentials)
            //snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots/')
        }
    }
}

signing {
    sign publishing.publications.mavenJava
}

tasks.register('lookupURLs', JavaExec) {
    dependsOn compileJava
    description = 'Reads the current Common Crawl URL index data and extracts all URLs for interesting mime-types or file extensions'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.index.DownloadURLIndex'
    classpath = sourceSets.main.runtimeClasspath
}


tasks.register('downloadOldIndex', JavaExec) {
    dependsOn compileJava
    description = 'Reading blocks at beginning or the last stored block-number and download binary data from the common crawl archives'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.oldindex.ReadAndDownload'
    classpath = sourceSets.main.runtimeClasspath
}

tasks.register('downloadDocuments', JavaExec) {
    dependsOn compileJava
    description = 'Uses the URLs listed in commoncrawl.txt to download the documents from the Common Crawl'

    jvmArgs = [
            '-Xmx128m'
    ]

    mainClass = 'org.dstadler.commoncrawl.index.DownloadFromCommonCrawl'
    classpath = sourceSets.main.runtimeClasspath
}

tasks.register('deduplicate', JavaExec) {
    dependsOn compileJava
    description = 'Compares files by size and hash and moves away identical files into a backup-directory'

    jvmArgs = [
            '-Xmx1024m'
    ]

    mainClass = 'org.dstadler.commoncrawl.Deduplicate'
    classpath = sourceSets.main.runtimeClasspath
}