Clone for gitlab

git_cloner/boot/boot.py

 """Naval Fate.
 
 Usage:
-  git-cloner clon [(-u <user> | --user=<user>) [(-p <pass>| --pass=<pass>)]] [(-f | --force)] [(-V | --verbose)] [(-o <out>| --out=<out>)]
-                  [(-r <file>| --retry=<file>)] ((-c <org> | --org=<org>) | (--U <user> | --target-user=<user>))
-                  (-t <type> | --type=<type>)
-                  [(-s <file> | --save=<file>)] [--ssh]
+  git-cloner clon [(-u <user> | --user=<user>) [(-p <pass>| --pass=<pass>)]] [--priv-token=<token>] [(-f | --force)]
+                  [(-V | --verbose)] [(-o <out>| --out=<out>)] [(-r <file>| --retry=<file>)]
+                  ((-c <org> | --org=<org>) | (--U <user> | --target-user=<user>)) (-t <type> | --type=<type>)
+                  [-H <host> | --host=<host>] [(-s <file> | --save=<file>)] [--ssh [<port>]]
   git-cloner make-repos [(-u <user> | --user=<user>)] [-f] [(-i <in>| --in=<in>)] [(-s <file> | --save=<file>)]
-                  (-t <type> | --type=<type>) --priv-token=<token> (-h | --host=<host>) [--ssh]
+                  (-t <type> | --type=<type>) --priv-token=<token> (-h <host>| --host=<host>) [--ssh]
   git-cloner (-h | --help)
   git-cloner --version
 
@@ -18,17 +18,17 @@ Options:
   -V --verbose            Show version.
   -c --org=<org>          Origin (organization) to read repositories. It is incompatible with target-user
   -h --help               Show help
+  -H --host=<host>        Host of repository (Not needed for github)
   -f --force              Force remove directories
   -o --out=<out>          Output directory
   -i --in=<in>            Input directory
   -s --save=<file>        Save Output result into a file
   -r --retry=<file>       Retry the for the last execution, the input is te save file from the previous execution
-  --ssh                   Use ssh instead https
-  -t --type=<type>        Indicate the type of repository gitlab is only supported jet
+  --ssh                   Use ssh instead https, 22 as default
+  <port>                  Port for ssh, default 22
+  -t --type=<type>        Indicate the type of repository. values: gitlab and github
   -U --target-user=<user> Origin (user) to read repositories. It is incompatible with org
   --priv-token=<token>    Indicate the path to the private token where the token is
-  -H --host=<host>        Indicate the host. i.e git@gitlab.com:4123
-
 
 """
 

git_cloner/gitlab_wrapper.py

@@ -34,7 +34,7 @@ def get_namespace():
     org = Config.get_config().org
     url = "http://git.fhuertas.com:55080/api/v3/namespaces"
     headers = {'PRIVATE-TOKEN': Config.get_config().token}
-    r = [{'id': 4, 'name': 'stratio', 'path': 'stratio', 'kind': 'group', 'full_path': 'stratio', 'parent_id': None,
+    r = [{'id': 4, 'name': 'sstt', 'path': 'sstt', 'kind': 'group', 'full_path': 'sstt', 'parent_id': None,
           'members_count_with_descendants': 1},
          {'id': 3, 'name': 'personal', 'path': 'personal', 'kind': 'group', 'full_path': 'personal', 'parent_id': None,
           'members_count_with_descendants': 1},

git_cloner/model/owner/github_owner.py

 from git_cloner.model.owner.owner import Owner
-from git_cloner.model.repository import RawRepository
+from git_cloner.model.repository import RawRepository, Repository
 from git_cloner.utils import utils
 from git_cloner.utils.config import Config
 
@@ -28,7 +28,7 @@ class GithubOwner(Owner):
         return self.REPOS_FROM_USER_TEMPLATE.format(base=self.api_base_url())
 
     def get_repos(self):
-        return [RawRepository(repo) for repo in self.get_repos_rec(0, self.get_repos_url())]
+        return [RawRepository.from_raw_data(repo, Repository.GITHUB) for repo in self.get_repos_rec(0, self.get_repos_url())]
 
     # Is not the best form but i'm, trying tailrec
     @staticmethod

git_cloner/model/owner/gitlab_owner.py

+from git_cloner.model.owner.owner import Owner
+from git_cloner.model.repository import RawRepository, Repository
+from git_cloner.utils.config import Config
+from git_cloner.utils import utils
+
+
+class GitlabOwner(Owner):
+    REPOSITORY_TYPE = "gitlab"
+
+    NAMESPACE_ENDPOINT = "{host}/api/v4/namespaces?search={owner}"
+    REPOS_USER_ENDPOINT = "{host}/api/v4/users/{ns}/projects"
+    REPOS_GROUPS_ENDPOINT = "{host}/api/v4/groups/{ns}/projects"
+
+    def __init__(self, owner, is_user, host):
+        self.host = host
+        self.owner = owner
+        self.is_user = is_user
+        self.namespace = self._get_ns(owner, host)
+
+    @classmethod
+    def _get_ns(cls, owner, host):
+        results = utils.make_request(cls.NAMESPACE_ENDPOINT.format(host=host, owner=owner))
+        raw = [result for result in results if result['name'] == owner][0]
+        return raw['id']
+
+    @staticmethod
+    def build_from_config():
+        conf = Config.get_config()
+        owner, is_user = (conf.target_user, True) if conf.target_user else (conf.org, False)
+        host = conf.host
+        return GitlabOwner(owner, is_user, host)
+
+    def get_repos(self):
+        url = self.REPOS_USER_ENDPOINT if self.is_user else self.REPOS_GROUPS_ENDPOINT
+        result = utils.make_request(url.format(host=self.host, ns=self.namespace))
+        return [RawRepository.from_raw_data(repo, Repository.GITLAB) for repo in result]

git_cloner/model/owner/owner.py

+from abc import abstractmethod
+
+
+
 class Owner(object):
+    @property
+    @abstractmethod
+    def REPOSITORY_TYPE(self):
+        raise NotImplementedError("Please Implement this method")
+
+    @abstractmethod
     def get_repos(self):
         raise NotImplementedError("Please Implement this method")
 
     @staticmethod
+    @abstractmethod
     def build_from_config():
         raise NotImplementedError("Please Implement this method")
 
     @staticmethod
     def builder(repo_type):
         from git_cloner.model.owner.github_owner import GithubOwner
+        from git_cloner.model.owner.gitlab_owner import GitlabOwner
         from git_cloner.model.repository import Repository
 
         if repo_type == Repository.GITHUB: return GithubOwner.build_from_config()
+        if repo_type == Repository.GITLAB: return GitlabOwner.build_from_config()
         raise RuntimeError("Invalid repository type")

git_cloner/model/remotes/gitlab_remote.py

 from uuid import uuid4
 
 from git_cloner.model.remotes.remote import Remote
+import re
 
 
 class GitlabRemote(Remote):
     SSH_URL_BASE = "ssh://git@{host}/{owner}/{repo}"
-    HTTP_URL_BASE = "http://{host}/{owner}/{repo}"
-    HTTPS_URL_BASE = "https://{host}/{owner}/{repo}"
+    HTTP_URL_BASE = "{host}/{owner}/{repo}"
 
-    def __init__(self, host, owner, repo_name, remote_type, name=uuid4()):
+    def __init__(self, host, owner, repo_name, is_ssh, port, name=uuid4()):
+        self.port = port
         self.name = name
         self.host = host
         self.owner = owner
         self.repo_name = repo_name
-        self.type = remote_type
+        self.is_ssh = is_ssh
 
     def get_name(self):
         return self.name
@@ -25,8 +26,12 @@ class GitlabRemote(Remote):
         return self.owner
 
     def build_remote_url(self):
-        return {
-            'ssh': self.SSH_URL_BASE.format(host=self.host, owner=self.owner, repo=self.repo_name),
-            'http': self.HTTP_URL_BASE.format(host=self.host, owner=self.owner, repo=self.repo_name),
-            'https': self.HTTPS_URL_BASE.format(host=self.host, owner=self.owner, repo=self.repo_name),
-        }.get(self.type)
+        if self.is_ssh:
+            split = self.host.split("//")[1].split(":", 1)
+            new_host = "{}:{}{}".format(
+                split[0],
+                self.port,
+                re.sub('^[\d]*', '', "" if len(split) == 1 else split[1]))
+            return self.SSH_URL_BASE.format(host=new_host, owner=self.owner, repo=self.repo_name, port=self.port)
+        else:
+            return self.HTTP_URL_BASE.format(host=self.host, owner=self.owner, repo=self.repo_name)

git_cloner/model/repository.py

@@ -5,7 +5,6 @@ import copy
 import os
 import shutil
 
-from git_cloner.model.remotes.github_remote import GithubRemote
 from git_cloner.utils.logger import *
 
 
@@ -59,11 +58,21 @@ class Repository(object):
 
 
 class RawRepository(object):
-    def __init__(self, raw_data):
-        self.raw_data = raw_data
+    def __init__(self, remote):
+        self.remote = remote
+
+    @staticmethod
+    def from_raw_data(raw_data, my_type):
+        config = Config.get_config()
+        if my_type == Repository.GITHUB:
+            owner = raw_data['owner']['login']
+            repo_name = raw_data['name']
+            return RawRepository(GithubRemote(owner, repo_name, config.ssh))
+        if my_type == Repository.GITLAB:
+            owner = raw_data['namespace']['name']
+            repo_name = raw_data['name']
+            return RawRepository(GitlabRemote(config.host, owner, repo_name, config.ssh, config.port))
+        raise RuntimeError("Invalid repository type")
 
     def clone(self, path):
-        owner = self.raw_data['owner']['login']
-        repo_name = self.raw_data['name']
-        remote = GithubRemote(owner, repo_name, Config.get_config().ssh)
-        return Repository.clone(remote, path)
+        return Repository.clone(self.remote, path)

git_cloner/operations/mass_cloner.py

 import json
 
 from git_cloner.model.owner.owner import Owner
-from git_cloner.model.repository import RawRepository
+from git_cloner.model.repository import RawRepository, Repository
+from git_cloner.utils import logger
 from git_cloner.utils.config import Config
 
 
@@ -20,12 +21,8 @@ def get_repos():
 
 
 def log_retry_path(path):
-    template_dic = """{{"name": "{name}","owner": {{"login": "{login}"}}}}"""
     with open(path) as file:
         lines = [line.strip() for line in file.readlines()]
         filtered = [line for line in lines if line.endswith("False")]
-        format_str = [template_dic.format(
-            name=line.split(",")[0].split(":")[1].split("/")[1].strip(),
-            login=line.split(",")[0].split(":")[1].split("/")[0].strip())
-            for line in filtered]
-        return [RawRepository(json.loads(line)) for line in format_str]
+        return [RawRepository(logger.build_remote_from_log_line(line)) for line in filtered]
+

git_cloner/utils/config.py

@@ -25,10 +25,11 @@ class Config(object):
     _KEY_SSH = "--ssh"
     _KEY_VERBOSE = "--verbose"
     _KEY_RETRY = "--retry"
+    _KEY_PORT_SSH = "<port>"
     instance = None
 
     def __init__(self, user, password, org, limit, dir_in, dir_out,
-                 force, save, ssh, repo_type, token, host, target_user, verbose, retry):
+                 force, save, ssh, port, repo_type, token, host, target_user, verbose, retry):
         self.retry = retry
         self.target_user = target_user
         self.user = user
@@ -40,6 +41,7 @@ class Config(object):
         self.force = force
         self.save = save
         self.ssh = ssh
+        self.port = port
         self.type = repo_type
         self.token = token
         self.host = host
@@ -55,6 +57,8 @@ class Config(object):
         force = arguments.pop(cls._KEY_FORCE, False)
         save = arguments.pop(cls._KEY_SAVE, None)
         ssh = arguments.pop(cls._KEY_SSH, False)
+        port = arguments.pop(cls._KEY_PORT_SSH, 22)
+        port = 22 if port is None else port
         repo_type = arguments.pop(cls._KEY_TYPE, None)
         host = arguments.pop(cls._KEY_HOST, None)
         path = arguments.pop(cls._KEY_TOKEN, None)
@@ -66,7 +70,7 @@ class Config(object):
             password = getpass.getpass()
 
         cls.instance = Config(user, password, org, 100, dir_in, dir_out,
-                              force, save, ssh, repo_type, token, host, target_user, verbose, retry)
+                              force, save, ssh, port, repo_type, token, host, target_user, verbose, retry)
         return cls.instance
 
     @classmethod

git_cloner/utils/logger.py

 import logging
 
+from git_cloner.model.remotes.github_remote import GithubRemote
+from git_cloner.model.remotes.gitlab_remote import GitlabRemote
 from git_cloner.utils.config import Config
 
 
@@ -16,7 +18,19 @@ def get_logger_console(origin_module,
 
 
 def log_result(remote, operation, result):
-    template = "\nRepo: {}/{}, Operation: {}, Result: {}"
-    if Config.get_config().save is not None:
+    template = "\nServer: {}, Repo: {}/{}, Operation: {}, Result: {}"
+    config = Config.get_config()
+    if config.save is not None:
         with open(Config.get_config().save, "a") as my_file:
-            my_file.write(template.format(remote.owner, remote.repo_name, operation, result))
+            my_file.write(template.format(config.type, remote.owner, remote.repo_name, operation, result))
+
+
+def build_remote_from_log_line(line):
+    server = line.split(",")[0].split(":")[1].strip()
+    owner = line.split(",")[1].split(":")[1].split("/")[0].strip()
+    name = line.split(",")[1].split(":")[1].split("/")[1].strip()
+    config = Config.get_config()
+    from git_cloner.model.repository import Repository
+    if server == Repository.GITHUB: return GithubRemote(owner, name, config.ssh)
+    if server == Repository.GITLAB: return GitlabRemote(config.host,owner, name, config.ssh)
+    raise RuntimeError("Invalid repository type")

git_cloner/utils/utils.py

@@ -14,6 +14,8 @@ def make_request(url, headers=None, auth_if_possible=True):
         headers = {}
     config = Config.get_config()
     auth = (config.user, config.password) if config.user and auth_if_possible else None
+    if config.token is not None:
+        headers['PRIVATE-TOKEN'] = config.token
     result = requests.get(url, auth=auth, headers=headers)
     if result.status_code != 200:
         raise RuntimeError("Incorrect request {}\n{}".format(url, str(result.content)))

tests/git_cloner/boot/boot_at.py

@@ -2,7 +2,7 @@
  "--help": False,
  "--host": None,
  "--in": None,
- "--org": "stratio",
+ "--org": "sstt",
  "--out": None,
  "--priv-token": None,
  "--retry": None,
@@ -14,15 +14,15 @@
  "--version": False,
  "clon": True,
  "make-repos": False}"""
-import unittest
 import runpy
-
 import sys
+import unittest
+
+from pkg_resources import resource_filename
 
 from git_cloner.boot import boot
 from git_cloner.utils.utils import read
 from tests.git_cloner.utils.utils import *
-from pkg_resources import resource_filename
 
 test_dir = "/tmp/test_tmp"
 
@@ -38,7 +38,6 @@ class BootAT(unittest.TestCase):
         sys.argv = self.args
         restore_path(self.old_dir)
 
-    # Command: git-cloner clon -u user -p pass -U yunxao --ssh -t github -o /tmp/test_tmp/fhuertas -s /tmp/test_tmp/log.log
     def test_run_clone_at(self):
         print("Command")
         print("git-cloner clon -u user -p pass -U yunxao --ssh -t github -o /tmp/test_tmp/fhuertas "
@@ -47,8 +46,29 @@ class BootAT(unittest.TestCase):
         password = read(os.path.join(secrets_path, 'password'))
         cmd = "git-cloner clon -u {user} -p {password} -U yunxao --ssh -t github -o {out_dir}/fhuertas " \
               "-s {out_dir}/log.log".format(out_dir=test_dir, user=user, password=password)
+        print(cmd)
         sys.argv = cmd.split(" ")
         runpy.run_module(boot.__name__, run_name="__main__", alter_sys=False)
         with open(os.path.join(test_dir, "log.log")) as file:
             lines = len([line for line in file.readlines() if line.strip() != ""])
             self.assertEqual(lines, 10)
+
+    def test_run_clone_from_gitlab_at(self):
+        test_group = read(os.path.join(secrets_path, 'personal_group_test'))
+        host = read(os.path.join(secrets_path, 'gitlab_host'))
+        ssh_port = read(os.path.join(secrets_path, 'gitlab_port'))
+        token = os.path.join(secrets_path, 'token_gitlab_personal')
+        cmd = "git-cloner clon --priv-token={token} --org={test_group} --ssh {port} " \
+              "-t gitlab -o {out_dir}/fhuertas -s {out_dir}/log.log -H {host}" \
+            .format(token=token,
+                    port=ssh_port,
+                    test_group=test_group,
+                    out_dir=test_dir,
+                    host=host)
+
+        print(cmd)
+        sys.argv = cmd.split(" ")
+        runpy.run_module(boot.__name__, run_name="__main__", alter_sys=False)
+        with open(os.path.join(test_dir, "log.log")) as file:
+            lines = len([line for line in file.readlines() if line.strip() != ""])
+            self.assertEqual(lines, 1)

tests/git_cloner/model/operations/mass_cloner_it.py

@@ -5,6 +5,7 @@ import pkg_resources
 from mock_decorators.function_mock import FunctionMock, FunctionMockResult
 
 from git_cloner.model.owner.github_owner import GithubOwner
+from git_cloner.model.remotes.github_remote import GithubRemote
 from git_cloner.model.repository import Repository
 from git_cloner.operations.mass_cloner import *
 from git_cloner.utils import utils
@@ -17,7 +18,7 @@ response_filepath = pkg_resources.resource_filename("tests", "resources/request_
 log_retry_path = pkg_resources.resource_filename("tests", "resources/retry.log")
 
 
-class MassClonerTest(unittest.TestCase):
+class MassClonerIT(unittest.TestCase):
     def setUp(self):
         self.dir = set_test_dir(test_dir)
         time.sleep(1)
@@ -26,7 +27,7 @@ class MassClonerTest(unittest.TestCase):
         restore_path(self.dir)
         time.sleep(1)
 
-    def serial_it(self):
+    def test_serial_it(self):
         def test_mass_cloner_user():
             config = Config.load_config({})
             config.type = "github"
@@ -67,7 +68,7 @@ class MassClonerTest(unittest.TestCase):
 
         def test_logger():
             config = Config.load_config({})
-            config.type = "github"
+            config.type = Repository.GITHUB
             config.org = "fhuertasorgfortest"
             config.ssh = True
             config.save = os.path.join(test_dir, "log.log")
@@ -76,7 +77,7 @@ class MassClonerTest(unittest.TestCase):
             repos_raw = [
                 generate_repo_github(name="monkey", user="fhuertasorgfortest"),
                 generate_repo_github()]
-            repos = [RawRepository(repo) for repo in repos_raw]
+            repos = [RawRepository.from_raw_data(repo, config.type) for repo in repos_raw]
 
             @FunctionMockResult(entity=GithubOwner, function_name="get_repos", result=repos)
             def inner_test():
@@ -87,8 +88,9 @@ class MassClonerTest(unittest.TestCase):
             with open(config.save) as file:
                 result = "".join(file.readlines()).strip()
                 self.assertEqual(result,
-                                 "Repo: fhuertasorgfortest/monkey, Operation: clone, Result: True\n"
-                                 "Repo: {}, Operation: clone, Result: False".format(repos_raw[1].get('full_name')))
+                                 "Server: github, Repo: fhuertasorgfortest/monkey, Operation: clone, Result: True\n"
+                                 "Server: github, Repo: {}, Operation: clone, Result: False".format(
+                                     repos_raw[1].get('full_name')))
 
         def test_get_repos_normal():
             config = Config.load_config({})
@@ -107,10 +109,12 @@ class MassClonerTest(unittest.TestCase):
             result = get_repos()
             self.assertEqual(len(result), 2)
 
-            self.assertEqual(result[0].raw_data['name'], 'monkey')
-            self.assertEqual(result[0].raw_data['owner']['login'], 'fhuertasorgfortest')
-            self.assertEqual(result[1].raw_data['name'], 'mix')
-            self.assertEqual(result[1].raw_data['owner']['login'], 'mix')
+            self.assertTrue(isinstance(result[0].remote, GithubRemote))
+            self.assertEqual(result[0].remote.repo_name, 'monkey')
+            self.assertEqual(result[0].remote.owner, 'fhuertasorgfortest')
+            self.assertTrue(isinstance(result[1].remote, GithubRemote))
+            self.assertEqual(result[1].remote.repo_name, 'mix')
+            self.assertEqual(result[1].remote.owner, 'mix')
 
         def test_retry_mass_clone():
             config = Config.load_config({})
@@ -130,10 +134,11 @@ class MassClonerTest(unittest.TestCase):
         self.tearDown()
         self.setUp()
         test_mass_cloner_fail()
-        self.setUp()
         self.tearDown()
+        self.setUp()
         test_logger()
         test_get_repos_normal()
+        self.setUp()
         test_get_repos_retry()
         self.tearDown()
         self.setUp()

monkey @ 7b68cc95

-Subproject commit 7b68cc95ef2a31611b76b6146cdee7bb7413f013

tests/git_cloner/model/owner/github_owner_test.py

 import unittest
+
 from mock_decorators.function_mock import FunctionMock
 
-from git_cloner.utils import utils
 from git_cloner.model.owner.github_owner import GithubOwner
+from git_cloner.utils import utils
 from git_cloner.utils.config import Config
 from tests.git_cloner.utils.gen_utils import *
 
@@ -38,6 +39,8 @@ class GithubOwnerTest(unittest.TestCase):
 
         @FunctionMock(utils, 'make_request', get_repos, False)
         def inner_test():
+            from git_cloner.model.owner.github_owner import GithubOwner
+
             owner = GithubOwner("fhuertas", True)
             return owner.get_repos()
 
@@ -63,3 +66,5 @@ class GithubOwnerTest(unittest.TestCase):
 
         result = inner_test()
         self.assertEqual(len(result), 0)
+
+

tests/git_cloner/model/owner/gitlab_owner_it.py

+import unittest
+
+import os
+from pkg_resources import resource_filename
+
+from git_cloner.model.owner.gitlab_owner import GitlabOwner
+from git_cloner.model.owner.owner import Owner
+from git_cloner.model.repository import Repository
+from git_cloner.utils.config import Config
+from git_cloner.utils.utils import read
+
+secrets_path = os.path.join(resource_filename('tests', 'resources'), 'secrets')
+user = read(os.path.join(secrets_path, 'user'))
+host = read(os.path.join(secrets_path, 'gitlab_host'))
+org = read(os.path.join(secrets_path, 'personal_group_test'))
+
+
+class GitlabOwnerIT(unittest.TestCase):
+    def test_builder_it(self):
+        config = Config.load_config({})
+        config.token = read(os.path.join(secrets_path, 'token_gitlab'))
+        owner = GitlabOwner(user, True, "https://gitlab.com")
+        self.assertEqual(owner.namespace, 2130450)
+
+    def test_builder_unauthorized_it(self):
+        Config.load_config({})
+        self.assertRaises(RuntimeError, GitlabOwner._get_ns, user, "https://gitlab.com")
+
+    def test_builder_personal_it(self):
+        config = Config.load_config({})
+        config.token = read(os.path.join(secrets_path, 'token_gitlab_personal'))
+        owner = GitlabOwner(user, True, host)
+        self.assertEqual(owner.namespace, 2)
+
+    def test_build_gitlab_it(self):
+        config = Config.load_config({})
+
+        config.host = host
+        config.target_user = user
+        config.token = read(os.path.join(secrets_path, 'token_gitlab_personal'))
+
+        owner = Owner.builder(Repository.GITLAB)
+        self.assertTrue(isinstance(owner, GitlabOwner))
+        self.assertEqual(owner.owner, user)
+        self.assertEqual(owner.host, host)
+        self.assertEqual(owner.namespace, 2)
+
+    def test_get_repos_user_it(self):
+        config = Config.load_config({})
+
+        config.host = host
+        config.org = org
+        config.token = read(os.path.join(secrets_path, 'token_gitlab_personal'))
+        owner = Owner.builder(Repository.GITLAB)
+
+        result = owner.get_repos()
+
+        self.assertEqual(len(result), 1)
+        print(result)

tests/git_cloner/model/owner/owner_test.py

 import unittest
 
+from mock_decorators.function_mock import FunctionMockResult
+from git_cloner.utils import utils
 from git_cloner.model.owner.github_owner import GithubOwner
+from git_cloner.model.owner.gitlab_owner import GitlabOwner
 from git_cloner.model.owner.owner import Owner
+from git_cloner.model.repository import Repository
 from git_cloner.utils.config import Config
 
 
 class OwnerTest(unittest.TestCase):
     def test_builder_github(self):
         Config.load_config({}).target_user = "pepe"
-        result = Owner.builder("github")
+        result = Owner.builder(Repository.GITHUB)
         self.assertEqual(result.owner, "pepe")
         self.assertEqual(result.is_user, True)
         self.assertTrue(isinstance(result, GithubOwner))
 
+    @FunctionMockResult(utils, "make_request", [{"name": "user", "id": "1234"}])
+    def test_build_gitlab(self):
+        config = Config.load_config({})
+
+        config.host = "http://host"
+        config.target_user = "user"
+
+        owner = Owner.builder(Repository.GITLAB)
+        self.assertTrue(isinstance(owner, GitlabOwner))
+        self.assertEqual(owner.owner, "user")
+        self.assertEqual(owner.host, "http://host")
+        self.assertEqual(owner.namespace, 1234)
+
     def test_builder_other(self):
         self.assertRaises(RuntimeError, Owner.builder, "invalid")
         pass

tests/git_cloner/model/remotes/git_remote_test.py → tests/git_cloner/model/remotes/github_remote_test.py

@@ -3,7 +3,7 @@ import unittest
 from git_cloner.model.remotes.github_remote import GithubRemote
 
 
-class GitRemoteTest(unittest.TestCase):
+class GithubRemoteTest(unittest.TestCase):
     def test_build_from_url_ssh(self):
         remote = GithubRemote.build_from_url("git@github.com:owner/repo-name.git")
         self.assertTrue(remote.ssh)

tests/git_cloner/model/remotes/gitlab_remote_test.py

@@ -5,16 +5,16 @@ from git_cloner.model.remotes.gitlab_remote import GitlabRemote
 
 class TestGitlabRemote(unittest.TestCase):
     def test_build_remote_ssh(self):
-        remote = GitlabRemote("git.host.com:2342", "owner", "repo", "ssh")
+        remote = GitlabRemote("http://git.host.com:2342", "owner", "repo", True)
         remote.build_remote_url()
         self.assertEqual(remote.build_remote_url(), "ssh://git@git.host.com:2342/owner/repo")
 
     def test_build_remote_http(self):
-        remote = GitlabRemote("git.host.com:2342", "owner", "repo", "http")
+        remote = GitlabRemote("http://git.host.com:2342", "owner", "repo", False)
         remote.build_remote_url()
         self.assertEqual(remote.build_remote_url(), "http://git.host.com:2342/owner/repo")
 
     def test_build_remote_https(self):
-        remote = GitlabRemote("git.host.com:2342", "owner", "repo", "https")
+        remote = GitlabRemote("https://git.host.com:2342", "owner", "repo", False)
         remote.build_remote_url()
         self.assertEqual(remote.build_remote_url(), "https://git.host.com:2342/owner/repo")

tests/git_cloner/model/remotes/remote_test.py

@@ -2,6 +2,7 @@ import unittest
 
 from git_cloner.model.remotes.github_remote import GithubRemote
 from git_cloner.model.remotes.gitlab_remote import GitlabRemote
+from tests.git_cloner.utils import gen_utils
 
 
 class TestRemote(unittest.TestCase):
@@ -10,9 +11,13 @@ class TestRemote(unittest.TestCase):
         self.assertEqual(remote.build_clone_cmd(), "git clone https://github.com/owner/repo-name.git {path}")
 
     def test_build_remote_add(self):
-        remote = GitlabRemote("git.host.com:2342", "owner", "repo", "ssh", "uuid")
-        self.assertEqual(remote.build_remote_add(), "git remote add uuid ssh://git@git.host.com:2342/owner/repo")
+        remote = GitlabRemote("http://git.host.com:2342", "owner", "repo", True, 1111, "uuid")
+        self.assertEqual(remote.build_remote_add(), "git remote add uuid ssh://git@git.host.com:1111/owner/repo")
+
+    def test_build_remote_add_with_path(self):
+        remote = GitlabRemote("http://git.host.com:2342/this/123", "owner", "repo", True, 1111, "uuid")
+        self.assertEqual(remote.build_remote_add(), "git remote add uuid ssh://git@git.host.com:1111/this/123/owner/repo")
 
     def test_build_remote_remove(self):
-        remote = GitlabRemote("git.host.com:2342", "owner", "repo", "ssh", "uuid")
+        remote = GitlabRemote("git.host.com:2342", "owner", "repo", True, name="uuid")
         self.assertEqual(remote.build_remote_delete(), "git remote remove uuid")

tests/git_cloner/model/repository_it.py

@@ -16,13 +16,13 @@ test_dir = "/tmp/test_tmp"
 class RepositoryIT(unittest.TestCase):
     def setUp(self):
         self.old_dir = set_test_dir(test_dir)
-        os.makedirs(test_dir)
 
     def tearDown(self):
-        restore_path(self.old_dir)  # , test_dir)
+        # restore_path(self.old_dir)  # , test_dir)
+        pass
 
     # This tests have problems if are executed in parallel
-    def serials_it(self):
+    def test_serials_it(self):
         def creating_with_cloning():
             remote = GithubRemote.build_from_url("https://github.com/fhuertas/scala_base.git")
             repo = Repository.clone(remote, test_dir)
@@ -43,16 +43,22 @@ class RepositoryIT(unittest.TestCase):
 
         def clone_repo_github():
             Config.load_config({}).ssh = True
-            gen_raw_repo = generate_repo_github()
-            gen_raw_repo['owner']['login'] = "fhuertas"
-            gen_raw_repo['name'] = "monkey"
-            raw_repo = RawRepository(gen_raw_repo)
+            gen_raw_repo = generate_repo_github("monkey", "fhuertas")
+            raw_repo = RawRepository(gen_raw_repo, Repository.GITHUB)
             result = raw_repo.clone(test_dir)
             self.assertTrue(result.check_git())
 
-            creating_with_cloning()
-            creating_with_cloning_ssh()
-            clone_repo_github()
+        def clone_repo_gitlab_ssh():
+            Config.load_config({}).ssh = True
+            gen_raw_repo = generate_repo_gitlab("test_project", "test_group")
+            raw_repo = RawRepository(gen_raw_repo, Repository.GITLAB)
+            result = raw_repo.clone(test_dir)
+            self.assertTrue(result.check_git())
+
+        # creating_with_cloning()
+        # creating_with_cloning_ssh()
+        clone_repo_github()
+        clone_repo_gitlab_ssh()
 
     def check_force_remove_dir_it(self):
         remote = GithubRemote.build_from_url("https://github.com/fhuertas/scala_base.git")
@@ -86,5 +92,3 @@ class RepositoryIT(unittest.TestCase):
         result = Repository.clone(remote, test_dir)
         self.assertIsNone(result)
         self.assertTrue(os.path.exists(target_path))
-
-

tests/git_cloner/test_gitlab_wrapper.py

@@ -12,7 +12,7 @@ class TestGitlabWrapper(unittest.TestCase):
         # push_repository("/tmp/tmp/git_1")
 
     def test_make_all(self):
-        Config.get_config().org = "stratio"
+        Config.get_config().org = "sstt"
         Config.get_config().host = "git.labs.com:1234"
         Config.get_config().token = utils.read('token')
         Config.get_config().dir_in = "/home/fhuertas/Projects/personal"

tests/git_cloner/test_module.py

@@ -3,23 +3,4 @@ from git_cloner import boot
 
 
 class TestModule(unittest.TestCase):
-    # def test_run(self):
-    #     arguments = {
-    #         '--org': 'stratio',
-    #         '--limit': '1'
-    #     }
-    #     boot.run(arguments)
-    # def test_run(self):
-    #     arguments = {
-    #         '--org': 'stratio',
-    #         '--limit': '100'
-    #     }
-    #     boot.run(arguments)
-
-    # def test_run(self):
-    #     arguments = {
-    #         '--org': 'stratios',
-    #         '--limit': '100'
-    #     }
-    #     boot.run(arguments)
     pass

tests/git_cloner/utils/config_test.py

@@ -63,3 +63,4 @@ class TestConfig(unittest.TestCase):
         self.assertEqual(my_config.target_user, None)
         self.assertEqual(my_config.verbose, False)
         self.assertEqual(my_config.retry, None)
+        self.assertEqual(my_config.port, 22)

tests/git_cloner/utils/gen_utils.py

@@ -10,6 +10,18 @@ def gen_name():
     return names[pos]
 
 
+def gen_host():
+    base = "http{s}://".format(s="s" if gen_bool() else "")
+    path = "/".join(gen_path())
+    return "{base}{path}".format(base=base, path=path)
+
+
+def gen_ssh_gitlab():
+    base = "ssh://git@".format(s="s" if gen_bool() else "")
+    path = "/".join(gen_path())
+    return "{base}{path}".format(base=base, path=path)
+
+
 def gen_int(start=0, limit=1000000):
     return randrange(start, limit)
 
@@ -18,6 +30,11 @@ def gen_bool():
     return bool(randrange(0, 2))
 
 
+def gen_path(max=4):
+    n = randrange(0, max)
+    return [gen_name() for _ in range(n)]
+
+
 def join_path(path1, path2): return "{}/{}".format(path1, path2)
 
 
@@ -121,3 +138,65 @@ def generate_repo_github(name=gen_name(), user=gen_name()):
         "watchers": 0,
         "default_branch": "master"
     }
+
+
+def generate_repo_gitlab(name=gen_name(), path=gen_name()):
+    host = gen_host()
+    ssh = gen_ssh_gitlab()
+    return {
+        'id': gen_int(),
+        'description': '',
+        'default_branch': None,
+        'tag_list': [],
+        'ssh_url_to_repo': '{ssh}/{path}/{name}.git'.format(ssh=ssh, path=path, name=name),
+        'http_url_to_repo': '{host}/{path}/{name}.git'.format(host=host, path=path, name=name),
+        'web_url': '{host}/{path}/{name}'.format(host=host, path=path, name=name),
+        'name': '{name}'.format(name=name),
+        'name_with_namespace': '{path} / {name}'.format(path=path, name=name),
+        'path': '{name}'.format(name=name),
+        'path_with_namespace': '{path}/{name}'.format(path=path, name=name),
+        'avatar_url': None,
+        'star_count': 0,
+        'forks_count': 0,
+        'created_at': '2017-11-09T01:47:09.917+01:00',
+        'last_activity_at': '2017-11-09T01:47:09.917+01:00',
+        '_links': {
+            'self': '{host}/api/v4/projects/4'.format(host=host),
+            'issues': '{host}/api/v4/projects/4/issues'.format(host=host),
+            'merge_requests': '{host}/api/v4/projects/4/merge_requests'.format(host=host),
+            'repo_branches': '{host}/api/v4/projects/4/repository/branches'.format(host=host),
+            'labels': '{host}/api/v4/projects/4/labels'.format(host=host),
+            'events': '{host}/api/v4/projects/4/events'.format(host=host),
+            'members': '{host}/api/v4/projects/4/members'.format(host=host)
+        },
+        'archived': gen_bool(),
+        'visibility': 'private',
+        'resolve_outdated_diff_discussions': gen_bool(),
+        'container_registry_enabled': gen_bool(),
+        'issues_enabled': gen_bool(),
+        'merge_requests_enabled': gen_bool(),
+        'wiki_enabled': gen_bool(),
+        'jobs_enabled': gen_bool(),
+        'snippets_enabled': gen_bool(),
+        'shared_runners_enabled': gen_bool(),
+        'lfs_enabled': gen_bool(),
+        'creator_id': gen_int(),
+        'namespace': {
+            'id': gen_int(),
+            'name': '{path}'.format(path=path),
+            'path': '{path}'.format(path=path),
+            'kind': 'group',
+            'full_path': '{path}'.format(path=path),
+            'parent_id': None,
+            'members_count_with_descendants': gen_int()
+        },
+        'import_status': 'none',
+        'open_issues_count': 0,
+        'public_jobs': gen_bool(),
+        'ci_config_path': None,
+        'shared_with_groups': [],
+        'only_allow_merge_if_pipeline_succeeds': gen_bool(),
+        'request_access_enabled': gen_bool(),
+        'only_allow_merge_if_all_discussions_are_resolved': gen_bool(),
+        'printing_merge_request_link_enabled': gen_bool()
+    }

tests/git_cloner/utils/logger_test.py

+import unittest
+
+from git_cloner.model.remotes.github_remote import GithubRemote
+from git_cloner.model.remotes.gitlab_remote import GitlabRemote
+from git_cloner.utils import logger
+from git_cloner.utils.config import Config
+
+
+class LoggerTest(unittest.TestCase):
+    def test_build_remote_from_log_line_github(self):
+        line = "Server: github, Repo: fhuertasorgfortest/monkey, Operation: clone, Result: False"
+        Config.load_config({}).ssh = True
+        remote = logger.build_remote_from_log_line(line)
+        self.assertTrue(isinstance(remote, GithubRemote))
+        self.assertEqual(remote.owner, "fhuertasorgfortest")
+        self.assertEqual(remote.repo_name, "monkey")
+        self.assertEqual(remote.ssh, True)
+
+
+    def test_build_remote_from_log_line_gitlab(self):
+        line = "Server: gitlab, Repo: fhuertasorgfortest/monkey, Operation: clone, Result: False"
+        config = Config.load_config({})
+        config.ssh = False
+        config.host = "https://this.is.a/host"
+        remote = logger.build_remote_from_log_line(line)
+        self.assertTrue(isinstance(remote, GitlabRemote))
+        self.assertEqual(remote.owner, "fhuertasorgfortest")
+        self.assertEqual(remote.repo_name, "monkey")
+        self.assertEqual(remote.is_ssh, False)
+        self.assertEqual(remote.host,"https://this.is.a/host")

tests/resources/retry.log

-Repo: fhuertasorgfortest/monkey, Operation: clone, Result: False
-Repo: lower/mix, Operation: clone, Result: True
-Repo: mix/mix, Operation: clone, Result: False
\ No newline at end of file
+Server: github, Repo: fhuertasorgfortest/monkey, Operation: clone, Result: False
+Server: github, Repo: lower/mix, Operation: clone, Result: True
+Server: github, Repo: mix/mix, Operation: clone, Result: False

tests/resources/retry_gitlab.log

+Server: gitlab, Repo: fhuertasorgfortest/monkey, Operation: clone, Result: False
+Server: gitlab, Repo: lower/mix, Operation: clone, Result: True
+Server: gitlab, Repo: mix/mix, Operation: clone, Result: False