Skip to content

P
python-cloner
Commit 2286d61b by Francisco Huertas
Options

First commit

parents
Showing with 649 additions and 0 deletions
.gitignore 0 → 100644
# IntelliJ files
.idea
env
env-*
# Python temporal files
__pycache__
*.egg-info
*.pyc
.coverage
# package files
dist
build
LICENSE 0 → 100644
This diff is collapsed. Click to expand it.
Makefile 0 → 100644
PYTHON_VERSION=
PYTHON_ENV_TEST=env-test$(PYTHON_VERSION)
all: clean env test-versions package
env: env/bin/activate
env/bin/activate:
bin/env.sh
test:
bin/tests.sh $(PYTHON_VERSION)
test-clean:
rm -Rf env-*
clean:
bin/clean.sh
package: clean env
bin/package.sh
continuous-test:
bash bin/exec-continuous-test.sh
README.md 0 → 100644
# Python cloner
## Make commands
* `make env`. Make a environment
* `make clean`. Clean the elements
* `make test-clean`. Clean the tests
* `make test`. make the tests
* `make package`. build the artifact
* `make continuous-test TEST=<test folder> PACKAGE=<package> [ENV=<env folder>]`
## Optional environment var for makefile
* PYTHON_VERSION: The version of python. i.e. 3.4
bin/clean.sh 0 → 100755
#!/usr/bin/env bash
BASEDIR=`dirname $0`/..
rm -rf $BASEDIR/env* $BASEDIR/*.egg-info $BASEDIR/env-test* dist build .coverage coverage.xml
bin/continuous-test.sh 0 → 100755
#!/bin/bash
if [ "$1" = "--help" ]; then
echo -b "This command execute the tests of a python file. The tests are find using prefix and postfix indicated. This"
echo "command need a special package in feora it is named inotify-tools."
echo ""
echo "Usage:"
echo " continous-test.sh [--env[ <path-activate-virtualenv>]] [--src <path-to-source>] [--test <path-to-test>]"
echo " [--prefix <test-prefix>] [--postfix <test-postfix> [--filter <test-filter>] [--check]"
echo " continous-test.sh --check"
echo " continous-test.sh --help"
fi
if [ "$1" = "--check" ]; then
if ! rpm -qa | grep -qw inotify-tools; then
echo "Installing requisites"
sudo dnf install inotify-tools
fi
exit 0
fi
if [[ $# == 1 ]]
then
if [[ "$1" == "-h" ]]
then
echo -n " $0 [--env[ <path-activate-virtuaenv>]] [--src <path-to-source>] [--test <path-to-test>] "
echo "[--prefix <test-prefix>] [--postfix <test-postfix> [--filter <test-filter>]"
exit 0
fi
fi
# Colores
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
# fichero de fuentes para inspeccionar
SOURCE_DIR='.'
# ficheros de test para buscar el test
TEST_DIR='.'
# prefijo y postfijo del test
PREFIX="test_"
POSTFIX=""
# comando del test %s irá el nombre del test
TEST_COMMAND="python -m unittest %s"
# Filtro posix para ignorar ficheros. Este esta puesto para ignorar los ficheros temporales de intellij
FILTER="___$"
# Operaciones a observar, 'attrib,modify' para intellij
WATCH_OP='attrib,modify'
n_arg=$#
for ((n_arg=1; n_arg<=$#;n_arg++))
do
arg=${!n_arg}
case ${arg} in
--env)
((n_arg++))
arg=${!n_arg}
if [[ "$arg" == --* ]]
then
env=env/bin/activate
((n_arg--))
else
env=${arg}/bin/activate
fi
printf "Activating virtual env: "
printf "$env\n"
set -e
source ${env}
set +e
;;
--src)
((n_arg++))
SOURCE_DIR=${!n_arg}
;;
--test)
((n_arg++))
TEST_DIR=${!n_arg}
;;
--prefix)
((n_arg++))
PREFIX=${!n_arg}
;;
--postfix)
((n_arg++))
PREFIX=${!n_arg}
;;
--filter)
((n_arg++))
PREFIX=${!n_arg}
;;
esac
done
echo "Configuration: "
if [ -n "$env" ]; then
echo " * Virtualenv: $env"
fi
echo " * source dir: $SOURCE_DIR"
echo " * test dir: $TEST_DIR"
echo " * file name test prefix: $PREFIX"
echo " * file name test postfix: $POSTFIX"
echo " * file filter: $FILTER"
# Cheking requisites
if [ ! -d ${TEST_DIR} -o ! -d ${SOURCE_DIR} ]; then
echo "Some configuration is not correct (Source or test dir)"
exit 1
fi
function test_src (){
while :;do
#Waiting to modifcation
inotify_results=$(inotifywait -r -e ${WATCH_OP} --exclude ${FILTER} ${SOURCE_DIR} 2>/dev/null)
results=$(echo ${inotify_results} | grep py$ | cut -d ' ' -f 3 | rev | cut -d '/' -f 1 | cut -d '.' -f 2- | rev )
n_test=0
n_test_failed=0
tests_failed=()
for file_name in $results
do
test_file_name=$PREFIX$file_name$POSTFIX".py"
test_path=$(find ${TEST_DIR} | grep ${test_file_name}$ )
if [ "$test_path" != "" ]; then
echo ""
echo " * File modified: '$file_name', searching tests with the name '$test_file_name'"
echo " * Running tests: "
printf '%s\n' "${test_path[@]}"
echo ""
echo "--------------START TESTS----------------"
echo ""
for test_file in $test_path
do
my_test_file=${test_file#./}
test_command=$(printf "$TEST_COMMAND\n" "${my_test_file}")
echo "---------------------"
echo "TEST: '$test_command'"
echo "---------------------"
eval ${test_command}
result=$?
n_test=$((n_test+1))
sleep 0.1
if [ ${result} -ne 0 ]; then
n_test_failed=$((n_test_failed+1))
tests_failed+=($test_file)
fi
done
echo ""
echo "--------------END TESTS----------------"
fi
done
if [ ${n_test} -ne 0 ]; then
echo "Results: "
DATE=`date +"%Y-%m-%dT%H:%M:%S"`
echo "Tests faileds: $n_test_failed/$n_test"
if [ ${n_test_failed} -eq 0 ]; then
printf "${GREEN}TESTS: OK${NC} (${DATE})\n"
else
printf ' %s\n' "${tests_failed[@]}"
printf "${RED}TESTS: FAIL${NC} (${DATE})\n"
fi
fi
done
}
function test_tests {
while :;do
#Waiting to modifcation
inotify_results=$(inotifywait -r -e ${WATCH_OP} --exclude ${FILTER} ${TEST_DIR} 2>/dev/null)
echo $inotify_results
results=$(echo ${inotify_results} | cut -d ' ' --output-delimiter='' -f 1,3)
results=$(echo ${results} | grep py$)
n_test=0
n_test_failed=0
tests_failed=()
for file_name in $results
do
if [ "$file_name" != "" ]; then
echo ""
echo " * File modified: '$file_name', Running test:"
echo ""
echo "--------------START TESTS----------------"
echo ""
test_command=$(printf "$TEST_COMMAND\n" "${file_name}")
echo "---------------------"
echo "TEST: '$test_command'"
echo "---------------------"
eval ${test_command}
result=$?
n_test=$((n_test+1))
if [ ${result} -ne 0 ]; then
n_test_failed=$((n_test_failed+1))
tests_failed+=(${test_file})
fi
echo ""
echo "--------------END TESTS----------------"
fi
done
if [ ${n_test} -ne 0 ]; then
echo "Results: "
DATE=`date +"%Y-%m-%dT%H:%M:%S"`
echo "Tests faileds: $n_test_failed/$n_test"
sleep 0.1
if [ ${n_test_failed} -eq 0 ]; then
printf "${GREEN}TESTS: OK${NC} (${DATE})\n"
else
printf ' %s\n' "${tests_failed[@]}"
printf "${RED}TESTS: FAIL${NC} (${DATE})\n"
fi
fi
done
}
function input {
while :;do
read input
if [ "$input" == "exit" ];
then
file=$(echo $0 | rev | cut -d '/' -f1 | rev)
$(killall ${file} inotifywait)
exit 0
elif [ "$input" == "test" ];
then
test_path="$TEST_DIR/$PREFIX*$POSTFIX"
temp="/"
test_path=${test_path/\/\//${temp}}
echo $test_path
cmd=$(printf "$TEST_COMMAND\n" "${test_path}")
eval ${cmd}
fi
done
}
test_src &
test_tests &
input
bin/env.sh 0 → 100755
#!/bin/bash -e
BASEDIR=`dirname $0`/..
source "${BASEDIR}/bin/var-env.sh"
PYTHON_VERSION=${PYTHON_VERSION:=${DEVELOP_PYTHON_VERSION}}
echo "Making env for python ${PYTHON_VERSION}"
virtualenv -p python${DEVELOP_PYTHON_VERSION} -q ${BASEDIR}/env
source $BASEDIR/env/bin/activate
pip install -r $BASEDIR/requirements.txt
pip install -e $BASEDIR
bin/exec-continuous-test.sh 0 → 100644
#!/usr/bin/env bash
BASEDIR=`dirname $0`/..
source $BASEDIR/bin/var-env.sh
ENV=${ENV:=env}
set -u
echo "[${PACKAGE}]: Executing tests (${TEST}). Environment: \"${ENV}\""
echo "${BASEDIR}/bin/continuous-test.sh --env ${ENV} --src ${PACKAGE} --test ${TEST}"
bash ${BASEDIR}/bin/continuous-test.sh --env ${ENV} --src ${PACKAGE} --test ${TEST}
bin/package.sh 0 → 100755
#!/bin/bash -e
BASEDIR=`dirname $0`/..
mkdir -p dist
echo "Building wheel..."
"$BASEDIR/env/bin/python" setup.py bdist_wheel
echo "Building egg..."
"$BASEDIR/env/bin/python" setup.py sdist
VERSION=`cat $BASEDIR/VERSION`
bin/tests.sh 0 → 100755
#!/bin/bash -e
BASEDIR=`dirname $0`/..
source "${BASEDIR}/bin/var-env.sh"
PYTHON_VERSION=${PYTHON_VERSION:=${DEVELOP_PYTHON_VERSION}}
VERBOSITY=${VERBOSITY:=$TEST_VERBOSITY}
TEST_DIR=${BASEDIR}/env-test${PYTHON_VERSION}
echo "Testing on python version: ${PYTHON_VERSION}"
echo "Test environment: $TEST_DIR"
if [ ! -d "$TEST_DIR" ]; then
virtualenv -p python$1 -q $TEST_DIR
echo "New virtualenv for UT created."
source $TEST_DIR/bin/activate
echo "New virtualenv for UT activated."
pip install -r $BASEDIR/requirements.txt
pip install -e $BASEDIR
fi
export COVERAGE_FILE=.coverage
for PACKAGE in ${PACKAGES[@]}; do
for TEST in ${TEST_FOLDERS[@]}; do
${TEST_DIR}/bin/nosetests --verbosity=${VERBOSITY} --with-coverage --cover-package=${PACKAGE} ${TEST}
done
done
$TEST_DIR/bin/coverage xml
echo "Total report"
$TEST_DIR/bin/coverage report
bin/var-env.sh 0 → 100755
#!/usr/bin/env bash
export DEVELOP_PYTHON_VERSION=3
export TEST_PYTHON_VERSION=(2.7 3 3.4 3.5)
export TEST_VERBOSITY=3
export PACKAGES=("github_cloner")
export TEST_FOLDERS=("tests")
\ No newline at end of file
github_cloner/boot.py 0 → 100644
"""Naval Fate.
Usage:
git-cloner clon [(-u <user> | --user=<user>)] [-f] [(-o <out>| --out=<out>)]
(-c <org> | --org=<org>) (-l <limit> | --limit=<limit>)
[(-s <file> | --save=<file>)] [--ssh]
git-cloner (-h | --help)
git-cloner --version
Options:
-l --limit=<limit> Limit of repositories
-u --user=<user> User to connect
-v --version Show version.
-c --org=<org> Origin to read repositories
-h --help Show help
-f Force remove directories
-o --out=<out> Output directory
-s --save=<file> Save Output resutl into a file
--ssh Use ssh instead https
"""
from docopt import docopt
from github_cloner.config import Config
from github_cloner.github_wrapper import clones
from github_cloner.github_wrapper import list_repos
def main():
arguments = docopt(__doc__, version='Git cloner 1.0')
run(arguments)
def run(arguments):
Config.load_config(arguments)
repos = list_repos(Config.org, Config.limit)
clones(repos)
github_cloner/config.py 0 → 100644
import getpass
class Config(object):
_KEY_USER = "--user"
_KEY_PASS = "--pass"
_KEY_ORG = "--org"
_KEY_LIMIT = "--limit"
_KEY_DIR = "--out"
_KEY_FORCE = "-f"
_KEY_SAVE = "--save"
_KEY_SSH = "--ssh"
# _KEY_OUTPUT =
user = None
password = None
org = None
limit = None
dir = "."
force = False
save = None
ssh = False
limit_per_page = 100
@classmethod
def load_config(cls, arguments):
cls.user = arguments.pop(cls._KEY_USER, None)
cls.password = arguments.pop(cls._KEY_PASS, None)
cls.org = arguments.pop(cls._KEY_ORG, None)
cls.limit = arguments.pop(cls._KEY_LIMIT, None)
cls.dir = arguments.pop(cls._KEY_DIR, ".")
cls.force = arguments.pop(cls._KEY_FORCE, False)
cls.save = arguments.pop(cls._KEY_SAVE, None)
cls.ssh = arguments.pop(cls._KEY_SSH, False)
if cls.user is not None and cls.password is None:
cls.password = getpass.getpass()
return cls
@classmethod
def get_config(cls):
return cls
github_cloner/github_wrapper.py 0 → 100644
import pathlib
import subprocess
import requests
import shutil
from github_cloner.config import Config
def list_repos(org, limit):
print(limit)
limit = int(limit)
total_pages = int(limit / int(Config.limit_per_page)) + 1
if limit % Config.limit_per_page:
total_pages += 1
print(total_pages)
return [repo for page in range(1, total_pages) for repo in list_repos_page(org, page)]
def list_repos_page(org, page):
KEYS = ['name', 'clone_url', 'ssh_url']
auth = (Config.user, Config.password) if Config.user else None
url = 'https://api.github.com/orgs/{org}/repos?page={page}&per_page={limit}'.format(
org=org, limit=Config.limit_per_page, page=page)
r = requests.get(url, auth=auth)
if r.status_code != 200:
print("Error {} in the request".format(r.status_code))
return []
repos = r.json()
filter_repos = [
{key: repo.get(key) for key in KEYS}
for repo in repos]
print(len(filter_repos))
return filter_repos
# return [{'name': 'dg-agent-commons', 'clone_url': 'https://github.com/Stratio/dg-agent-commons.git',
# 'ssh_url': 'git@github.com:Stratio/dg-agent-commons.git'},
# {'name': 'tiki-takka', 'clone_url': 'https://github.com/Stratio/tiki-takka.git',
# 'ssh_url': 'git@github.com:Stratio/tiki-takka.git'}]
def clones(repos):
out_dir = '.' if Config.dir is None else Config.dir
force = Config.force
size = len(repos)
repo_result = []
if out_dir is not None:
print(Config.dir)
print(Config.dir)
print(Config.dir)
pathlib.Path(out_dir).mkdir(parents=True, exist_ok=True)
print("Cloning repositories {}".format(len(repos)))
n = 1
for repo in repos:
repo_dir = "{}/{}".format(out_dir, repo.get("name"))
repo_name = repo.get("name")
clone_url = repo.get('ssh_url') if Config.ssh else repo.get('clone_url')
print("Clone repository {name} ({n}/{total})".format(n=n, total=size, name=repo_name))
cmd_str = 'git clone {clone_url} {dir}'.format(clone_url=clone_url, dir=repo_dir)
if force:
shutil.rmtree(repo_dir, ignore_errors=True)
cmd = cmd_str.split(" ")
try:
subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
result_str = " * {} Cloned".format(repo_name)
print(result_str)
repo_result.append(result_str)
except Exception:
result_str = " * {} CANNOT be cloned".format(repo_name)
print(result_str)
repo_result.append(result_str)
n += 1
if Config.save:
with open(Config.save, "w") as text_file:
text_file.write("\n".join(repo_result))
text_file.write("\n")
integration_files/.travis.yml 0 → 100644
language: python
python:
- "2.7"
- "3.3"
- "3.4"
- "3.5"
install:
- pip install coveralls
script:
- make test
after_success:
- coveralls
setup.cfg 0 → 100644
[metadata]
description-file = README.md
setup.py 0 → 100644
import os
from setuptools import setup
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read().strip()
required = [
'docopt==0.6.2',
'requests==2.18.4'
]
setup(name='git_cloner',
version=read('VERSION'),
author="Francisco Huertas",
author_email="francisco@fhuertas.com",
license="Apache2",
packages=["github_cloner"],
description="Python tool for clon repositories",
long_description=read('README.md'),
url='https://github.com/fhuertas/python_base',
download_url="https://github.com/fhuertas/python_base/tarball/{}".format(read('VERSION')),
install_requires=required,
classifiers=[
"Development Status :: 5 - Production/Stable",
"Topic :: Utilities",
"License :: OSI Approved :: Apache Software License",
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.5',
],
entry_points={
'console_scripts': [
'git-cloner=github_cloner.boot:main',
],
},)
tests/github_cloner/file.txt 0 → 100644
* dg-agent-commons CANNOT be cloned
* cassandra CANNOT be cloned
tests/github_cloner/test_github_wrapper.py 0 → 100644
import unittest
from github_cloner import github_wrapper
from github_cloner.config import Config
class TestModule(unittest.TestCase):
def test_run(self):
Config.dir = "pp"
Config.force = False
Config.save = "file.txt"
repos = [{'name': 'dg-agent-commons', 'clone_url': 'https://github.com/Stratio/dg-agent-commons.git',
'ssh_url': 'git@github.com:Stratio/dg-agent-commons.git'},
{'name': 'cassandra', 'clone_url': 'https://github.com/Stratio/cassandra.git',
'ssh_url': 'git@github.com:Stratio/cassandra.git'}]
github_wrapper.clones(repos)
# def test_run_no_dir(self):
# Config.force = False
# repos = [{'name': 'luke', 'clone_url': 'https://github.com/Stratio/luke.git',
# 'ssh_url': 'git@github.com:Stratio/luke.git'},
# {'name': 'cassandra', 'clone_url': 'https://github.com/Stratio/cassandra.git',
# 'ssh_url': 'git@github.com:Stratio/cassandra.git'}]
#
# github_wrapper.clones(repos)
tests/github_cloner/test_module.py 0 → 100644
import unittest
from github_cloner import boot
class TestModule(unittest.TestCase):
# def test_run(self):
# arguments = {
# '--org': 'stratio',
# '--limit': '1'
# }
# boot.run(arguments)
# def test_run(self):
# arguments = {
# '--org': 'stratio',
# '--limit': '100'
# }
# boot.run(arguments)
# def test_run(self):
# arguments = {
# '--org': 'stratios',
# '--limit': '100'
# }
# boot.run(arguments)
pass
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment