use https submodule urls by default

bin/breplay_stream.py

@@ -1,6 +1,10 @@
 #!/usr/bin/python3
 #
-# Replay a stream, as if mitmproxy were sending filter requests via the filter/apply API
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Replay a stream, as if mitmproxy were sending filter requests via the filter/apply API.
+#
+# This is an advanced tool for low-level debugging and diagnostics of the Bubble filtering system.
 #
 # Usage:
 #
@@ -13,9 +17,19 @@
 #
 # To capture requests for later playback:
 #   * Set debug_stream_fqdn and debug_stream_uri in mitmproxy/bubble_config.py and restart mitmproxy servers
-#   * Request a matching using a device whose traffic will be routed through the mitmproxy
+#   * Request the debug fqdn and URI using a device whose traffic will be routed through the mitmproxy
 #   * Capture files will be written to /tmp/bubble_stream_[request-id]_chunkXXXX.[url, headers.json, data]
 #
+# When capturing, the URL you request must match the debug_stream_fqdn and debug_stream_uri exactly
+# in order for capturing to be triggered. For example, if we have:
+#
+#   debug_stream_fqdn = 'www.example.com'
+#   debug_stream_uri = '/dir/test.html'
+#
+# Then the URL to use when requesting from the connected device is:
+#
+#   https://www.example.com/dir/test.html
+#
 import glob
 import json
 import requests

bin/create_user_and_network

@@ -1,58 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
-#
-#
-# Create a new user and start a network for that user
-#
-# Usage:  create_user_and_network <json-file>
-#
-#   json-file     : a JSON file containing specs for the new user and network
-#
-# The default values are:
-#
-#    {
-#      "rootUsername": "root",
-#      "rootPassword": "password",
-#      "sageFqdn": "_required",
-#      "nginxPort": "1443",
-#      "username": "user-<<rand 5>>",
-#      "password": "<<rand 10>>",
-#      "network": "test-net-{{rand 5}}",
-#      "email": "user-<<rand 5>>@example.com",
-#      "domain": "bubblev.org",
-#      "locale": "en_US",
-#      "lang": "en",
-#      "timezone": "EST",
-#      "plan": "bubble pro",
-#      "footprint": "Worldwide, excluding OFAC",
-#      "compute": "VultrCompute",
-#      "region": "New Jersey"
-#    }
-#
-# The required fields above have a value of "_required". A minimal example would be:
-#
-#    {
-#      "sageFqdn": "api-XXXXX.staging.bubblev.org"
-#    }
-#
-# This would add a user with a random name user-{{rand 5}} with a random password and random @example.com email address,
-# and a randomly named network with the name test-net-{{rand 5}}
-#
-# Environment variables
-#
-#   BUBBLE_API     : which API to use. Default is local (http://127.0.0.1:PORT, where PORT is found in .bubble.env)
-#   BUBBLE_USER    : account to use. Default is root
-#   BUBBLE_PASS    : password for account. Default is root
-#   BUBBLE_SCRIPTS : location of run.sh script. Default is to assume it is in the same directory containing this script
-#
-SCRIPT="${0}"
-SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
-. ${SCRIPT_DIR}/bubble_common
-
-JSON="${1:?no json provided}"
-if [[ ! -f "${JSON}" ]] ; then
-  die "json file not found: ${JSON}"
-fi
-
-cat "${JSON}" | exec ${SCRIPT_DIR}/bscript debug ${SCRIPT_DIR}/../scripts/create_user_and_network.json --no-login --vars -

bin/first_time_setup.sh

@@ -2,7 +2,6 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
-#
 # Performs first-time setup after a fresh git clone.
 # Installs utility libraries.
 #
@@ -16,8 +15,11 @@
 # ~/.bubble.env is the environment used by the BubbleServer started by run.sh
 # ~/.bubble-test.env is the environment used by the BubbleServer that runs during the integration tests
 #
+# If you prefer to checkout git submodules using SSH instead of HTTPS, set the BUBBLE_SSH_SUBMODULES
+# environment variable to 'true'
+#
 
-function die {
+function die() {
   if [[ -z "${SCRIPT}" ]] ; then
     echo 1>&2 "${1}"
   else
@@ -26,14 +28,18 @@ function die {
   exit 1
 }
 
-BASE=$(cd $(dirname $0)/.. && pwd)
-cd ${BASE}
+BASE="$(cd "$(dirname "${0}")/.." && pwd)"
+cd "${BASE}" || die "Error changing to ${BASE} directory"
+
+if [[ -z "${BUBBLE_SSH_SUBMODULES}" || "${BUBBLE_SSH_SUBMODULES}" != "true" ]] ; then
+  "${BASE}"/bin/git_https_submodules.sh || die "Error switching to HTTPS git submodules"
+fi
 
 git submodule update --init --recursive || die "Error in git submodule update"
 
-pushd utils/cobbzilla-parent
+pushd utils/cobbzilla-parent || die "Error pushing utils/cobbzilla-parent directory"
 mvn install || die "Error installing cobbzilla-parent"
-popd
+popd || die "Error popping back from utils/cobbzilla-parent"
 
 UTIL_REPOS="
 cobbzilla-parent
@@ -43,11 +49,11 @@ templated-mail-sender
 cobbzilla-wizard
 abp-parser
 "
-pushd utils
-for repo in ${UTIL_REPOS} ; do
-  pushd ${repo} && mvn -DskipTests=true -Dcheckstyle.skip=true clean install && popd || die "Error installing ${repo}"
+pushd utils || die "Error pushing utils directory"
+for repo in ${UTIL_REPOS}; do
+  pushd "${repo}" && mvn -DskipTests=true -Dcheckstyle.skip=true clean install && popd || die "Error installing ${repo}"
 done
-popd
+popd || die "Error popping back from utils directory"
 
 if [[ -z "${BUBBLE_SETUP_MODE}" || "${BUBBLE_SETUP_MODE}" == "web" ]] ; then
   INSTALL_WEB=web mvn -DskipTests=true -Dcheckstyle.skip=true clean package || die "Error building bubble jar"

bin/git_https_submodules.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Update .gitmodules file to use HTTPS URLs instead of SSH URLs for git submodules
+#
+SCRIPT_DIR="$(cd "$(dirname "${0}")" && pwd)"
+GIT_MODS="${SCRIPT_DIR}/../.gitmodules"
+GIT_TEMP_FILE="$(mktemp "${GIT_MODS}.XXXXXXX")"
+cat "${GIT_MODS}" | while read -r line ; do
+  if [[ $(echo "${line}" | grep -c 'url = ') -gt 0 && $(echo "${line}" | grep -c 'git@git') -gt 0 ]] ; then
+    REPO="$(echo "${line}" | awk -F ':' '{print $2}')"
+    echo "url = https://git.bubblev.org/${REPO}" | tee -a "${GIT_TEMP_FILE}"
+  else
+    echo "${line}" | tee -a "${GIT_TEMP_FILE}"
+  fi
+done
+mv "${GIT_TEMP_FILE}" "${GIT_MODS}"