Taskfile

library 'common'
node('w2v'){
	checkout scm

    stage 'download image'
    ecr.pull_image('parser', 'v0.4.5')
	sh "docker build . -t wiki"

    withDockerContainer(image:'wiki', args: '-u root:root') {
        sh 'pip install awscli'

        stage "download requirement"
        version = sh(returnStdout: true, script: "cat version").trim()
        langs = ["zh", "ja", "en"]
        date = sh(returnStdout: true, script: "date +%Y-%m-%d").trim()

        println(version)
        println(date)

        for (String lang: langs){
            stage(lang){
                stage "download ${lang}"
                sh "mkdir -p ${lang}"
                sh "python WikiDumper.py ${lang}"

                stage "prepare ${lang}"
                sh "cd ${lang} && ls -lh"
                sh "cd ${lang} && ls *.bz2 | xargs -i -t basename {} .bz2 | xargs -i -t ../WikiExtractor.py -b 50m --processes=32 {}.bz2 -o {} --lang ${lang}"

                stage "remove ${lang}"
                sh "rm ${lang}/*.bz2"

                stage "upload s3 ${lang}"
                try{
                    s3.rename("gliacloud-nlp", "rawdata/${lang}/${version}/lastest/wiki", "rawdata/${lang}/${version}/${date}/wiki")
                }catch(Exception ex){
                    println("Catching the exception");
                }
                s3.upload_folder("gliacloud-nlp", "${lang}", "rawdata/${lang}/${version}/lastest/wiki")
                sh "rm -rf ${lang}"
            }
        }
    }
}