Introduction
In this post, we will see a sample Jenkin Pipeline Groovy script, where we will build a python project, and run pylint, coverage tools and push to artifactory.
Setup of Python Project
Your project must have following files:
- setup.py
- version.txt
- requirements.txt
- Have tests folder with test cases
Sample version.txt
__version__ = '0.0.1'Sample setup.py
import setuptools
import os.path
with open("README.md", "r") as fh:
long_description = fh.read()
def read(rel_path):
here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, rel_path), 'r') as fp:
return fp.read()
def get_version(rel_path):
for line in read(rel_path).splitlines():
if line.startswith('__version__'):
delim = '"' if '"' in line else "'"
return line.split(delim)[1]
else:
raise RuntimeError("Unable to find version string.")
setuptools.setup(
name="your-project-id",
version=get_version('version.txt'),
author="<Author>",
author_email="<Author email>",
description="<Sample description>",
long_description=long_description,
long_description_content_type="text/markdown",
url="<Github url for your project>",
packages=setuptools.find_packages(),
test_suite="tests",
install_requires=['dependencies'],
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
],
python_requires='>=3.6',
)Note: You must have Jenkins installed with pipeline module
What My Groovy Script Does
- Triggers on PR (Pull Request) creation and merging on
mainbranch. - Install few dependencies like
virtualenv, coverage wheel pylintetc - Install project dependencies from
requirements.txt - Check existence of
version.txtfile - Run Pylint and fail under a threshold mentioned
- Run Unit tests and fail under a threshold coverage number
- Deploy it on artifactory as Python module so that you can install via
pip
Jenkin Groovy Script
def getValue(Map map, String key, def defaultValue) {
return map.containsKey(key) ? map.get(key) : defaultValue;
}
def call(String artifactType = 'python_lib_compilation', Map optionalConf = [:]) {
def valid_artifactTypes = ['python_lib_compilation'];
String mainBranchName = getValue(optionalConf, 'mainBranchName', 'main');
String slave_name = getValue(optionalConf, 'slave_name', 'Jenkins_Slave');
float warningsThresholdPylint = getValue(optionalConf, 'warningsThresholdPylint', 8.0);
int warningsThresholdCoverage = getValue(optionalConf, 'warningsThresholdCoverage', 70);
String writerServerCredId = getValue(optionalConf, 'writerServerCredId', 'my_writer_artifactory_api_key');
String readerServerCredId = getValue(optionalConf, 'readerServerCredId', 'my_reader_artifactory_api_key');
if (valid_artifactTypes.contains(artifactType)) {
pipeline {
agent {
node {
label slave_name
}
}
stages {
stage('PreSteps') {
parallel {
stage('MainBranchPreSteps') {
when {
anyOf {
branch mainBranchName;
changeRequest target: mainBranchName
}
}
steps {
echo 'This is ' + mainBranchName + ' Branch'
echo '====installing requirements.txt===='
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId:readerServerCredId,
usernameVariable: 'ARTIFACTORY_USERNAME', passwordVariable: 'ARTIFACTORY_PASSWORD']
]) {
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install --upgrade pip
python3.8 -m pip install coverage wheel pylint
python3.8 -m pip install -r requirements.txt -i https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD}@artifactory.my-corp.com/artifactory/api/pypi/pypi-my-corp-release/simple
deactivate
"""
}
echo '====installation done===='
}
}
stage('NonMainBranchPreSteps') {
when {
not {
anyOf {
branch mainBranchName;
changeRequest target: mainBranchName
}
}
}
steps {
echo 'This is Non ' + mainBranchName + ' Branch'
}
}
}
}
// only when to merge to master branch, so not on PR or when new branch is cut
stage('compilation') {
when {
not {
anyOf {
changeRequest target: mainBranchName
}
}
anyOf {
branch mainBranchName;
}
}
steps {
echo 'compilation..'
echo 'merging to main branch'
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install coverage wheel pylint
python3.8 -m pip install --upgrade pip
python3.8 setup.py sdist bdist_wheel
deactivate
"""
}
}
stage('Test-File-Existence') {
parallel {
// only when there is a PR
// testing if version.txt file exists or not
stage('Test-File-Exists') {
when {
allOf {
changeRequest target: mainBranchName
expression { return fileExists('version.txt') && readFile('version.txt').contains('__version__') }
}
}
steps {
echo "checking if version.txt file exists or not"
echo "=====file exists===="
}
}
stage('Test-File-Dont-Exists') {
when {
allOf {
changeRequest target: mainBranchName
}
not {
expression { return fileExists('version.txt') && readFile('version.txt').contains('__version__') }
}
}
steps {
echo "=====Either file does not exists or version is not defined===="
sh 'exit 1'
}
}
}
}
// only when there is a PR
// testing structure i.e. pylint, coverage, unittest
stage('Test-Structure') {
when {
anyOf {
changeRequest target: mainBranchName
}
}
parallel {
stage('Pylint-test') {
when {
anyOf {
changeRequest target: mainBranchName
}
}
steps {
echo 'Testing..'
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install --upgrade pip
python3.8 -m pip install coverage wheel pylint
python3.8 setup.py sdist bdist_wheel
echo "Pylint warning number: ${warningsThresholdPylint}"
pylint --fail-under=${warningsThresholdPylint} **/*.py>pylint.log
cat pylint.log
deactivate
"""
}
}
stage('Unit-test') {
when {
anyOf {
changeRequest target: mainBranchName
}
}
steps {
echo 'Testing..'
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install --upgrade pip
python3.8 -m pip install coverage wheel pylint
python3.8 -m unittest tests/*.py
deactivate
"""
}
}
stage('Coverage-test') {
when {
anyOf {
changeRequest target: mainBranchName
}
}
steps {
echo 'Testing..'
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install --upgrade pip
python3.8 -m pip install coverage wheel pylint
coverage run -m unittest tests/*.py
coverage report -m --fail-under=${warningsThresholdCoverage} **/*.py
deactivate
"""
}
}
}
}
// deploy only when merged
stage('Deploy') {
when {
allOf {
branch mainBranchName;
}
}
steps {
echo 'Deploying....'
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId:writerServerCredId,
usernameVariable: 'TWINE_USERNAME', passwordVariable: 'TWINE_PASSWORD']
]) {
sh """
python3.8 -m pip install virtualenv
python3.8 -m venv python_env
source python_env/bin/activate
python3.8 -m pip install --upgrade pip
python3.8 -m pip install coverage wheel pylint twine
python3.8 -m twine upload --repository-url https://artifactory.corp.adobe.com/artifactory/api/pypi/pypi-cloudtech-release-local dist/* -u ${TWINE_USERNAME} -p ${TWINE_PASSWORD}
deactivate
"""
}
}
}
}
post {
always {
emailext(subject: '$DEFAULT_SUBJECT', body: '$DEFAULT_CONTENT', attachLog: true, recipientProviders: [[$class: 'CulpritsRecipientProvider'], [$class: 'DevelopersRecipientProvider'], [$class: 'FailingTestSuspectsRecipientProvider'], [$class: 'FirstFailingBuildSuspectsRecipientProvider'], [$class: 'RequesterRecipientProvider'], [$class: 'BuildUserRecipientProvider']], to: '$DEFAULT_RECIPIENTS')
}
}
}
} else {
pipeline {
agent {
node {
label slave_name
}
}
stages {
stage('InvalidStage') {
steps {
echo "The build argument provided must one of : ${valid_artifactTypes.join(',')}"
}
}
}
}
}
}Let me know if you have any question, I will try to resolve your queries.
Enjoy reading…
$ subscribe --weekly
One deep dive a week, in your inbox
System design, AI engineering, trading systems and interview prep. Tested code, real numbers. Free, and you can unsubscribe any time.
or read it on gyanbyte.substack.com
