Add support for using SSH

This patch changes "create_config.py" script interface and its
behaviour:

* by default anonymous Gerrit is used for repository cloning and code
  URLS,
* external git mirror for code URLs might be provided but there is no
  default one,
* SSH credentials might be provided for repository cloning.

Cloning repository using SSH is required due to current ONAP
infrastructure HTTPS bandwidth constraints.

User wishing to use this setup should provide two environmental
variables:

1) API_USER - Gerrit username for cloning repositories via SSH
2) API_KEY - Gerrit API key (User menu -> Settings -> HTTP Credentials)

Be aware that a new RSA key will be generated and added to the user's
Gerrit account for purposes of Hound code search operation inside a VM.
Do remember to delete it (User menu -> Settings -> SSH keys) after
evaluating this environment.

Issue-ID: ONAPARC-540
Change-Id: I3e98dfa8582322d5b3bbbbf377d748faea0da57b
Signed-off-by: Pawel Wieczorek <p.wieczorek2@samsung.com>

diff --git a/bootstrap/codesearch/Vagrantfile b/bootstrap/codesearch/Vagrantfile
index 0407a57..774a021 100644 (file)
--- a/bootstrap/codesearch/Vagrantfile
+++ b/bootstrap/codesearch/Vagrantfile
@@ -5,6 +5,12 @@ host_ip = "192.168.121.1"
 synced_folder = "/vagrant"
 houndd_bin = "${HOME}/go/bin/houndd"
 houndd_config = "${HOME}/config.json"
+key_file = "${HOME}/.ssh/id_rsa"
+api_base = "https://gerrit.onap.org/r"
+api_user = ENV.fetch('API_USER') { |user| abort("missing env: #{user}") }
+api_key = ENV.fetch('API_KEY') { |key| abort("missing env: #{key}") }
+onap_git = "git.onap.org"
+gerrit_port = "29418"
 
 $replace_dns = <<-SCRIPT
   HOST_IP="$1"
@@ -12,11 +18,35 @@ $replace_dns = <<-SCRIPT
   echo nameserver "$HOST_IP" | tee /etc/resolv.conf
 SCRIPT
 
+$generate_key = <<-SCRIPT
+  KEY_FILE="$1"
+  echo "Generating SSH key (${KEY_FILE})"
+  ssh-keygen -q -b 4096 -t rsa -f "$KEY_FILE" -N ""
+SCRIPT
+
+$upload_key = <<-SCRIPT
+  KEY_FILE="$1"
+  API_BASE="$2"
+  echo "Uploading SSH pubkey (${KEY_FILE}.pub) for user: ${API_USER}"
+  curl -sS \
+       -u "${API_USER}:${API_KEY}" \
+       -d "@${KEY_FILE}.pub" \
+       -H "Content-Type: text/plain" \
+       -X POST "${API_BASE}/a/accounts/${API_USER}/sshkeys"
+SCRIPT
+
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/ubuntu1804"
   config.vm.synced_folder ".", synced_folder, type: "rsync", rsync__exclude: "Vagrantfile"
   config.vm.network "forwarded_port", guest: 6080, host: 6080
   config.vm.provision "replace_dns", type: :shell, run: "always", inline: $replace_dns, args: host_ip
+  config.vm.provision "generate_key", type: :shell, privileged: false, inline: $generate_key, args: key_file
+  config.vm.provision "upload_key", type: :shell do |s|
+    s.privileged = false
+    s.inline = $upload_key
+    s.args = [key_file, api_base]
+    s.env = {'API_USER': api_user, 'API_KEY': api_key}
+  end
   config.vm.provision "dependencies", type: :shell, inline: <<-SHELL
     export DEBIAN_FRONTEND=noninteractive
     apt-get update
@@ -26,9 +56,11 @@ Vagrant.configure("2") do |config|
     export GOPATH="${HOME}/go"
     go get -u github.com/hound-search/hound/cmds/...
   SHELL
-  config.vm.provision "generate_config", type: :shell, privileged: false, inline: <<-SHELL
-    python3 #{synced_folder}/create_config.py > #{houndd_config}
-  SHELL
+  config.vm.provision "generate_config", type: :shell do |s|
+    s.privileged = false
+    s.inline = "python3 #{synced_folder}/create_config.py --ssh ${1} ${2} --git ${3} > #{houndd_config}"
+    s.args = [api_user, gerrit_port, onap_git]
+  end
   config.vm.provision "run_codesearch", type: :shell, privileged: false, inline: <<-SHELL
     tmux new -d -s codesearch #{houndd_bin} -conf #{houndd_config}
   SHELL

diff --git a/bootstrap/codesearch/create_config.py b/bootstrap/codesearch/create_config.py
index aef99e7..115b467 100755 (executable)
--- a/bootstrap/codesearch/create_config.py
+++ b/bootstrap/codesearch/create_config.py
@@ -8,8 +8,8 @@ import json
 import urllib.request
 import sys
 
-DEFAULT_GERRIT = "https://gerrit.onap.org/r"
-DEFAULT_GIT = "https://git.onap.org"
+DEFAULT_GERRIT = "gerrit.onap.org"
+API_PREFIX = "/r"
 API_PROJECTS = "/projects/"
 
 MAGIC_PREFIX = ")]}'"
@@ -20,7 +20,7 @@ GIT_ANCHOR = "#n{line}"
 
 def get_projects_list(gerrit):
     """Request list of all available projects from ONAP Gerrit."""
-    resp = urllib.request.urlopen(gerrit + API_PROJECTS)
+    resp = urllib.request.urlopen("https://{}{}{}".format(gerrit, API_PREFIX, API_PROJECTS))
     resp_body = resp.read()
 
     no_magic = resp_body[len(MAGIC_PREFIX):]
@@ -30,23 +30,23 @@ def get_projects_list(gerrit):
     return projects.keys()
 
 
-def create_repos_list(projects, gitweb, git, gerrit):
+def create_repos_list(projects, gerrit, ssh, git):
     """Create a map of all projects to their repositories' URLs."""
-    gerrit_project_url = "{}/{}.git"
-    gitweb_code_url = "{}/gitweb?p={}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}"
-
-    git_project_url = "{}/{}"
-    git_code_url = "{url}/tree/{path}{anchor}"
+    gerrit_url = "https://{}{}".format(gerrit, API_PREFIX)
+    gerrit_project_url = "{}/{{}}.git".format(gerrit_url)
+    gitweb_code_url = "{}/gitweb?p={{}}.git;hb=HEAD;a=blob;f={{path}}{{anchor}}".format(gerrit_url)
 
     repos_list = {}
     for project in projects:
-        if gitweb:
-            project_url = gerrit_project_url.format(gerrit, project)
-            code_url = gitweb_code_url.format(gerrit, project)
-            anchor = GITWEB_ANCHOR
-        else:
-            project_url = git_project_url.format(git, project)
-            code_url = git_code_url
+        project_url = gerrit_project_url.format(project)
+        code_url = gitweb_code_url.format(project)
+        anchor = GITWEB_ANCHOR
+
+        if ssh and len(ssh) == 2:
+            user, port = ssh[0], ssh[1]
+            project_url = "ssh://{}@{}:{}/{}.git".format(user, gerrit, port, project)
+        if git:
+            code_url = "https://{}/{}/tree/{{path}}{{anchor}}".format(git, project)
             anchor = GIT_ANCHOR
 
         repos_list[project] = {
@@ -64,9 +64,8 @@ def parse_arguments():
     """Return parsed command-line arguments."""
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--gerrit', help='Gerrit address', default=DEFAULT_GERRIT)
-    access = parser.add_mutually_exclusive_group()
-    access.add_argument('--gitweb', help='use Gerrit\'s gitweb (bool)', action='store_true')
-    access.add_argument('--git', help='use external git (address)', default=DEFAULT_GIT)
+    parser.add_argument('--ssh', help='SSH information: user, port', nargs=2)
+    parser.add_argument('--git', help='external git address')
 
     return parser.parse_args()
 
@@ -76,7 +75,7 @@ def main():
     arguments = parse_arguments()
 
     projects = get_projects_list(arguments.gerrit)
-    repos = create_repos_list(projects, arguments.gitweb, arguments.git, arguments.gerrit)
+    repos = create_repos_list(projects, arguments.gerrit, arguments.ssh, arguments.git)
     config = {
         "max-concurrent-indexers": 2,
         "dbpath": "data",