build.gradle

/*
 * Licensed to Elasticsearch B.V. under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch B.V. licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

buildscript {
    repositories {
        mavenCentral()
        maven {
            url 'https://plugins.gradle.org/m2/'
        }
    }
    dependencies {
        classpath "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
    }
}

plugins {
    id "de.undercouch.download" version "4.0.4"
}

apply plugin: 'de.undercouch.download'
apply from: "rubyUtils.gradle"


import de.undercouch.gradle.tasks.download.Download
import groovy.json.JsonSlurper

allprojects {
  group = 'org.logstash'

  apply plugin: 'java'
  apply plugin: 'idea'
  apply plugin: 'java-library'

  project.sourceCompatibility = JavaVersion.VERSION_1_8
  project.targetCompatibility = JavaVersion.VERSION_1_8

  tasks.withType(JavaCompile).configureEach {
      options.compilerArgs.add("-Xlint:all")
      options.compilerArgs.add("-Xlint:-processing")
      options.compilerArgs.add("-Werror")
  }

  tasks.withType(Javadoc).configureEach {
      options.addStringOption("Xwerror", "-quiet")
      if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_9) > 0) {
          options.addBooleanOption("html5", true)
      }
  }

  tasks.withType(Copy).configureEach {
      duplicatesStrategy = DuplicatesStrategy.EXCLUDE
  }

  clean {
      delete "${projectDir}/out/"
  }

  //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
  tasks.withType(Test) {
    testLogging {
      // set options for log level LIFECYCLE
      events "passed", "skipped", "failed", "standardOut"
      showExceptions true
      exceptionFormat "full"
      showCauses true
      showStackTraces true
      enableAssertions false

      // set options for log level DEBUG and INFO
      debug {
        events "started", "passed", "skipped", "failed", "standardOut", "standardError"
        exceptionFormat "full"
      }
      info.events = debug.events
      info.exceptionFormat = debug.exceptionFormat

      afterSuite { desc, result ->
        if (!desc.parent) { // will match the outermost suite
          def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)"
          def startItem = '|  ', endItem = '  |'
          def repeatLength = startItem.length() + output.length() + endItem.length()
          println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
        }
      }
    }
  }
}

subprojects {
    repositories {
        mavenCentral()
        maven {
            url 'https://plugins.gradle.org/m2/'
        }
    }
    dependencies {
        implementation "gradle.plugin.com.github.jk1:gradle-license-report:0.7.1"
    }

    apply plugin: 'com.github.jk1.dependency-license-report'

    licenseReport {
        renderer = new com.github.jk1.license.render.CsvReportRenderer()
        configurations = ['compile', 'runtime']
    }
}

version = versionMap['logstash-core']
String artifactVersionsApi = "https://artifacts-api.elastic.co/v1/versions"

tasks.register("configureArchitecture") {
    String arch = System.properties['os.arch']
    String osName = System.properties['os.name']
    String beatsArch = arch
    String esArch = arch

    // For aarch64 architectures, beats and elasticsearch name their artifacts differently
    if (arch == "aarch64") {
        beatsArch="arm64"
        esArch="aarch64"
    } else if (arch == "amd64") {
        beatsArch="x86_64"
        esArch="x86_64"
    }

    if (osName ==~ /Mac OS X/) {
        osName = "darwin"
    } else {
        osName = "linux"
    }
    project.ext.set("beatsArchitecture", "${osName}-${beatsArch}")
    project.ext.set("esArchitecture", "${osName}-${esArch}")
}

tasks.register("configureArtifactInfo") {
    dependsOn configureArchitecture
    def versionQualifier = System.getenv('VERSION_QUALIFIER')
    if (versionQualifier) {
        version = "$version-$versionQualifier"
    }

    def isReleaseBuild = System.getenv('RELEASE') == "1" || versionQualifier
    String apiResponse = artifactVersionsApi.toURL().text

    def dlVersions = new JsonSlurper().parseText(apiResponse)
    String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
    if (qualifiedVersion == null) {
        throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
    }
    // find latest reference to last build
    String buildsListApi = "${artifactVersionsApi}/${qualifiedVersion}/builds/"
    apiResponse = buildsListApi.toURL().text
    def dlBuilds = new JsonSlurper().parseText(apiResponse)
    def stackBuildVersion = dlBuilds["builds"][0]

    project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
    project.ext.set("stackArtifactSuffix", qualifiedVersion)
}



// Tasks

clean {
  delete "${projectDir}/Gemfile"
  delete "${projectDir}/Gemfile.lock"
  delete "${projectDir}/vendor"
  delete "${projectDir}/.bundle"
  delete "${projectDir}/qa/integration/Gemfile.lock"
  delete "${projectDir}/qa/integration/.bundle"
  delete "${projectDir}/build/licenseReportFolders.txt"
  delete "${projectDir}/build/rubyDependencies.csv"
}

def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect {
  it.tasks.findByName("assemble")
}

tasks.register("installBundler") {
    dependsOn assemblyDeps
    outputs.files file("${projectDir}/vendor/bundle/jruby/2.5.0/bin/bundle")
    doLast {
      gem(projectDir, buildDir, "bundler", "~> 2", "${projectDir}/vendor/bundle/jruby/2.5.0")
  }
}

tasks.register("bootstrap"){
    dependsOn installBundler
    doLast {
      setupJruby(projectDir, buildDir)
  }
}


tasks.register("installDefaultGems") {
    dependsOn bootstrap
  doLast {
      rake(projectDir, buildDir, 'plugin:install-default')
  }
}

tasks.register("installTestGems") {
    dependsOn bootstrap
    doLast {
      rake(projectDir, buildDir, 'plugin:install-development-dependencies')
  }
}

tasks.register("compileGrammar") {
    dependsOn bootstrap
    doLast {
        rake(projectDir, buildDir, 'compile:grammar')
    }
}

tasks.register("assembleTarDistribution") {
  dependsOn bootstrap
  inputs.files fileTree("${projectDir}/rakelib")
  inputs.files fileTree("${projectDir}/bin")
  inputs.files fileTree("${projectDir}/config")
  inputs.files fileTree("${projectDir}/lib")
  inputs.files fileTree("${projectDir}/modules")
  inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
  inputs.files fileTree("${projectDir}/logstash-core/lib")
  inputs.files fileTree("${projectDir}/logstash-core/src")
  inputs.files fileTree("${projectDir}/x-pack")
  outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
  doLast {
      rake(projectDir, buildDir, 'artifact:tar')
  }
}

tasks.register("assembleOssTarDistribution") {
  dependsOn bootstrap
  inputs.files fileTree("${projectDir}/rakelib")
  inputs.files fileTree("${projectDir}/bin")
  inputs.files fileTree("${projectDir}/config")
  inputs.files fileTree("${projectDir}/lib")
  inputs.files fileTree("${projectDir}/modules")
  inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
  inputs.files fileTree("${projectDir}/logstash-core/lib")
  inputs.files fileTree("${projectDir}/logstash-core/src")
  doLast {
      rake(projectDir, buildDir, 'artifact:tar_oss')
  }
}

tasks.register("assembleZipDistribution") {
  dependsOn bootstrap
  inputs.files fileTree("${projectDir}/rakelib")
  inputs.files fileTree("${projectDir}/bin")
  inputs.files fileTree("${projectDir}/config")
  inputs.files fileTree("${projectDir}/lib")
  inputs.files fileTree("${projectDir}/modules")
  inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
  inputs.files fileTree("${projectDir}/logstash-core/lib")
  inputs.files fileTree("${projectDir}/logstash-core/src")
  inputs.files fileTree("${projectDir}/x-pack")
  outputs.files file("${buildDir}/logstash-${project.version}.zip")
  doLast {
      rake(projectDir, buildDir, 'artifact:zip')
  }
}

tasks.register("assembleOssZipDistribution") {
  dependsOn bootstrap
  inputs.files fileTree("${projectDir}/rakelib")
  inputs.files fileTree("${projectDir}/bin")
  inputs.files fileTree("${projectDir}/config")
  inputs.files fileTree("${projectDir}/lib")
  inputs.files fileTree("${projectDir}/modules")
  inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
  inputs.files fileTree("${projectDir}/logstash-core/lib")
  inputs.files fileTree("${projectDir}/logstash-core/src")
  outputs.files file("${buildDir}/logstash-${project.version}.zip")
  doLast {
      rake(projectDir, buildDir, 'artifact:zip_oss')

  }
}

project(":logstash-core") {
  ["rubyTests", "test"].each { tsk ->
    tasks.getByPath(":logstash-core:" + tsk).configure {
      dependsOn installTestGems
    }
  }
}

def logstashBuildDir = "${buildDir}/logstash-${project.version}-SNAPSHOT"

tasks.register("unpackTarDistribution", Copy) {
  dependsOn assembleTarDistribution
  def tar = file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
  inputs.files tar
  outputs.files fileTree(logstashBuildDir)
  from tarTree(tar)
  into {buildDir}
}

def qaVendorPath = "${buildDir}/qa/integration/vendor"
def qaBundledGemPath = "${qaVendorPath}/jruby/2.5.0"
def qaBundleBin = "${qaBundledGemPath}/bin/bundle"

tasks.register("installIntegrationTestBundler"){
    dependsOn unpackTarDistribution
    outputs.files file("${qaBundleBin}")
  doLast {
      gem(projectDir, buildDir, "bundler", "~> 2", qaBundledGemPath)
  }
}

tasks.register("installIntegrationTestGems") {
  dependsOn installIntegrationTestBundler
  inputs.files file("${projectDir}/qa/integration/Gemfile")
  inputs.files file("${projectDir}/qa/integration/integration_tests.gemspec")
  inputs.files file("${logstashBuildDir}/Gemfile")
  inputs.files file("${logstashBuildDir}/Gemfile.lock")
  inputs.files file("${logstashBuildDir}/logstash-core/logstash-core.gemspec")
  outputs.files fileTree("${qaVendorPath}")
  outputs.files file("${projectDir}/qa/integration/Gemfile.lock")
  doLast {
      bundleWithEnv(
        projectDir, buildDir,
        "${projectDir}/qa/integration", qaBundleBin, ['install', '--path', qaVendorPath],
        [LS_GEM_PATH: qaBundledGemPath, LS_GEM_HOME: qaBundledGemPath]
      )
  }
}

tasks.register("downloadFilebeat", Download) {
    dependsOn  configureArtifactInfo
    description "Download Filebeat Snapshot for current branch version: ${version}"

    project.ext.set("versionFound", true)

    String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
    project.ext.set("unpackedFilebeatName", downloadedFilebeatName)

    // find url of build artifact
    String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
    String apiResponse = artifactApiUrl.toURL().text
    def buildUrls = new JsonSlurper().parseText(apiResponse)

    project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
    project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")


    src project.ext.filebeatSnapshotUrl
    onlyIfNewer true
    inputs.file("${projectDir}/versions.yml")
    outputs.file(project.ext.filebeatDownloadLocation)
    dest new File(project.ext.filebeatDownloadLocation)
    retries 3
    doLast {
        System.out.println "Downloaded to ${project.ext.filebeatDownloadLocation}"
    }
}

tasks.register("deleteLocalFilebeat", Delete) {
    delete ('./build/filebeat')
}

tasks.register("copyFilebeat", Copy){
    dependsOn = [downloadFilebeat, deleteLocalFilebeat]
    from tarTree(resources.gzip(project.ext.filebeatDownloadLocation))
    into "./build/"
    doLast {
        file("./build/${project.ext.unpackedFilebeatName}").renameTo('./build/filebeat')
        System.out.println "Unzipped ${project.ext.filebeatDownloadLocation} to ./build/filebeat"
        System.out.println "Deleting ${project.ext.filebeatDownloadLocation}"
        delete(project.ext.filebeatDownloadLocation)
    }
}


tasks.register("downloadEs", Download) {
    dependsOn  configureArtifactInfo
    description "Download ES Snapshot for current branch version: ${version}"

    String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
    project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")

    // find url of build artifact
    String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
    String apiResponse = artifactApiUrl.toURL().text
    def buildUrls = new JsonSlurper().parseText(apiResponse)

    project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
    project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")

    src project.ext.elasticsearchSnapshotURL
    onlyIfNewer true
    retries 3
    inputs.file("${projectDir}/versions.yml")
    outputs.file(project.ext.elasticsearchDownloadLocation)
    dest new File(project.ext.elasticsearchDownloadLocation)

    doLast {
        System.out.println "Downloaded to ${project.ext.elasticsearchDownloadLocation}"
    }
}


tasks.register("deleteLocalEs", Delete) {
    delete ('./build/elasticsearch')
}

tasks.register("copyEs", Copy){
    dependsOn = [downloadEs, deleteLocalEs]
    from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation))
    into "./build/"
    doLast {
        file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch')
        System.out.println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch"
        System.out.println "Deleting ${project.ext.elasticsearchDownloadLocation}"
        delete(project.ext.elasticsearchDownloadLocation)
    }
}

def rubyIntegrationSpecs = project.hasProperty("rubyIntegrationSpecs") ? ((String) project.property("rubyIntegrationSpecs")).split(/\s+/).join(",") : "specs/**/*_spec.rb"
def integrationTestPwd = "${projectDir}/qa/integration"

project(":logstash-integration-tests") {
    tasks.getByPath(":logstash-integration-tests:integrationTests").configure {
        systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs
        environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG')
        workingDir integrationTestPwd
        dependsOn installIntegrationTestGems
    }
}

tasks.register("runIntegrationTests"){
    dependsOn tasks.getByPath(":logstash-integration-tests:integrationTests")
    dependsOn copyEs
    dependsOn copyFilebeat
    shouldRunAfter ":logstash-core:test"
}



tasks.register("generateLicenseReport", JavaExec) {
    dependsOn generateLicenseReportInputs
    dependsOn ":dependencies-report:assemble"

    def jarFile = project('dependencies-report').getBuildDir().toString() + "/libs/dependencies-report.jar"

    String licenseReportInputCSV = project.hasProperty("licenseReportInputCSV") ? project.property("licenseReportInputCSV") : "build/dependencies.csv.ruby"
    String licenseReportOutputCSV = project.hasProperty("licenseReportOutputCSV") ? project.property("licenseReportOutputCSV") : "build/dependencies.csv"
    String noticePath = "NOTICE.txt"

    classpath = project.files([jarFile])
    main = "org.logstash.dependencies.Main"
    args licenseReportInputCSV,
      project.getBuildDir().toString() + "/licenseReportFolders.txt",
      licenseReportOutputCSV, noticePath
}

tasks.register("generateLicenseReportInputs") {
    dependsOn subprojects.generateLicenseReport

    // write location of all license reports for subprojects containing artifacts that are distributed to single file
    StringBuilder licenseReportFolders = new StringBuilder()
    subprojects.findAll { s1 -> !s1.hasProperty("isDistributedArtifact") || s1.property("isDistributedArtifact") == 'true'}.each { s ->
        s.tasks.findAll { t2 -> t2.getName() == "generateLicenseReport" }.each { t3 ->
            licenseReportFolders.append(t3.outputs.files.asPath + "\n")
        }
    }

    if (gradle.startParameter.taskNames.contains("generateLicenseReport")) {
        def licenseReportPath = project.getBuildDir().toString() + "/licenseReportFolders.txt"
        def licenseReportFolder = new File(licenseReportPath)
        licenseReportFolder.delete()
        licenseReportFolder = new File(licenseReportPath)
        licenseReportFolder.createNewFile()
        if (licenseReportFolder.canWrite()) {
            licenseReportFolder.text = licenseReportFolders.toString()
        }
    }
}

tasks.register("generatePluginsVersion") {
  dependsOn installDefaultGems
  doLast {
      rake(projectDir, buildDir, 'generate_plugins_version')
  }
}

bootstrap.dependsOn assemblyDeps
// FIXME: adding the integration tests task to check will mean
// that any registered task will be evaluated. This creates an issue
// where the downloadES task may throw an error on versions where
// Elasticsearch doesn't yet have a build we can fetch
// So for now we'll remove this to unblock builds, but finding a way
// to compartimentalize failures is needed going forward
//check.dependsOn runIntegrationTests

Boolean oss = System.getenv('OSS').equals('true')

if (!oss) {
  project(":logstash-xpack") {
    ["rubyTests", "rubyIntegrationTests", "test"].each { tsk ->
      tasks.getByPath(":logstash-xpack:" + tsk).configure {
        dependsOn installTestGems
      }
    }
  }
}

 tasks.register("runXPackUnitTests"){
   dependsOn ":logstash-xpack:rubyTests"
 }
 tasks.register("runXPackIntegrationTests"){
   dependsOn ":logstash-xpack:rubyIntegrationTests"
 }