diff --git a/bin/build_dist b/bin/build_dist index e1f55803..db65f5da 100755 --- a/bin/build_dist +++ b/bin/build_dist @@ -52,9 +52,32 @@ ZIP="${DIST_BASE}/bubble-${VERSION}.zip" mkdir -p "${DIST}" || die "Error creating distribution directory: ${DIST}" cp "${JAR}" "${DIST}/bubble.jar" || die "Error copying ${JAR} to ${DIST}/bubble.jar" cp "${BASE}/README.md" "${DIST}/README.md" || die "Error copying README.md to ${DIST}/README.md" +cp "${BASE}/LICENSE.md" "${DIST}/LICENSE.md" || die "Error copying LICENSE.md to ${DIST}/LICENSE.md" cp -R "${BASE}/docs" "${DIST}" || die "Error copying docs directory to ${DIST}" cp -R "${BASE}/bin" "${DIST}" || die "Error copying bin directory to ${DIST}" cp -R "${BASE}/config" "${DIST}" || die "Error copying config directory to ${DIST}" cd "${DIST}/.." && zip -r "${ZIP}" "$(basename ${DIST})" echo "Distribution created: " ls -lh "${ZIP}" + +if [[ ! -z "${BUBBLE_DIST_HOME}" ]] ; then + IS_DEV=0 + if [[ -z ${BUILD_NUMBER} ]] ; then + BUILD_NUMBER="dev" + IS_DEV=1 + fi + BUBBLE_VERSION="${VERSION}.${BUILD_NUMBER}" + + BUBBLE_DIST_TOP=${BUBBLE_DIST_HOME}/releases/bubble + BUBBLE_DIST=${BUBBLE_DIST_TOP}/${BUBBLE_VERSION}/$(basename ${ZIP}) + BUBBLE_DIST_DIR="$(dirname ${BUBBLE_DIST})" + if [[ ! -d "${BUBBLE_DIST_DIR}" ]] ; then + mkdir -p ${BUBBLE_DIST_DIR} + fi + cp "${ZIP}" "${BUBBLE_DIST}" && cat "${ZIP}" | sha256sum | cut -f1 -d' ' | tr -d '\n' > "${BUBBLE_DIST}.sha256" + if [[ ${IS_DEV} -eq 0 ]] ; then + cd ${BUBBLE_DIST_TOP} && rm -f latest && ln -sf ${BUBBLE_VERSION} latest + echo "${BUBBLE_VERSION}" > latest.txt + fi + echo "Published release: ${BUBBLE_DIST}" +fi diff --git a/bin/first_time_ubuntu.sh b/bin/first_time_ubuntu.sh index f01ba4b1..2e335e18 100755 --- a/bin/first_time_ubuntu.sh +++ b/bin/first_time_ubuntu.sh @@ -7,6 +7,16 @@ function die { exit 1 } +function db_user_exists { + username="${1}" + num_users="$(echo "select count(*) from pg_user where usename='${username}'" | su - postgres psql -qt | egrep -v '^$')" + if [[ -z "${num_users}" || ${num_users} -eq 0 ]] ; then + echo "0" + else + echo "1" + fi +} + # Ensure system is current sudo apt update -y || die "Error running apt update" sudo apt upgrade -y || die "Error running apt upgrade" @@ -21,7 +31,11 @@ BUBBLE_BIN="$(cd "$(dirname "${0}")" && pwd)" # Create DB user for current user, as superuser CURRENT_USER="$(whoami)" -sudo su - postgres bash -c 'createuser -U postgres --createdb --createrole --superuser '"${CURRENT_USER}"'' || die "Error creating ${CURRENT_USER} DB user" +if [[ $(db_user_exists ${CURRENT_USER}) ]] ; then + echo "PostgreSQL user ${CURRENT_USER} already exists, not creating" +else + sudo su - postgres bash -c 'createuser -U postgres --createdb --createrole --superuser '"${CURRENT_USER}"'' || die "Error creating ${CURRENT_USER} DB user" +fi PG_HBA=$(find /etc/postgresql -mindepth 1 -maxdepth 1 -type d | sort | tail -1)/main/pg_hba.conf sudo cat ${PG_HBA} | sed -e 's/ peer/ trust/g' | sed -e 's/ md5/ trust/g' > /tmp/pg_hba.conf || die "Error filtering ${PG_HBA}" @@ -30,6 +44,3 @@ sudo service postgresql restart || die "Error restarting pgsql" # Create DB user 'bubble', with the ability to create databases createuser --createdb bubble || die "Error creating bubble DB user" - -# Create bubble database -createdb --encoding=UTF-8 bubble || die "Error creating bubble DB" diff --git a/bin/install_packer.sh b/bin/install_packer.sh index dacc0d79..37741365 100755 --- a/bin/install_packer.sh +++ b/bin/install_packer.sh @@ -9,7 +9,7 @@ function die { # Install packer if [[ ! -f ${HOME}/packer/packer ]] ; then - PACKER_VERSION=1.5.6 + PACKER_VERSION=1.6.2 PACKER_FILE=packer_${PACKER_VERSION}_linux_amd64.zip PACKER_URL=https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_FILE} mkdir -p ${HOME}/packer && cd ${HOME}/packer && wget ${PACKER_URL} && unzip ${PACKER_FILE} || die "Error installing packer" @@ -19,7 +19,7 @@ fi # Install packer Vultr plugin if [[ ! -f ${HOME}/.packer.d/plugins/packer-builder-vultr ]] ; then - PACKER_VULTR_VERSION=1.0.8 + PACKER_VULTR_VERSION=1.0.11 PACKER_VULTR_FILE=packer-builder-vultr_${PACKER_VULTR_VERSION}_linux_64-bit.tar.gz PACKER_VULTR_URL=https://github.com/vultr/packer-builder-vultr/releases/download/v${PACKER_VULTR_VERSION}/${PACKER_VULTR_FILE} mkdir -p ${HOME}/.packer.d/plugins && cd ${HOME}/.packer.d/plugins && wget ${PACKER_VULTR_URL} && tar xzf ${PACKER_VULTR_FILE} || die "Error installing packer vultr plugin" diff --git a/bin/reset_bubble_logs b/bin/reset_bubble_logs new file mode 100755 index 00000000..77d55dfb --- /dev/null +++ b/bin/reset_bubble_logs @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# +# Truncate all bubble logs +# +BUBBLE_LOGS_DIR=/var/log/bubble +if [[ ! -d ${BUBBLE_LOGS_DIR} ]] ; then + echo "BUBBLE_LOGS_DIR not found, nothing to truncate: ${BUBBLE_LOGS_DIR}" + exit 0 +fi + +find ${BUBBLE_LOGS_DIR} -type f | while read log ; do + cat /dev/null > ${log} && echo "truncated ${log}" || echo "error truncating ${log}" +done diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java index 3230efa4..0340d73e 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java @@ -8,18 +8,18 @@ import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX; public abstract class PackerImageParserBase extends ListResourceParser { - private String bubbleVersion; - private String keyHash; + private final String bubbleVersion; + private final String versionHash; - public PackerImageParserBase(String bubbleVersion, String keyHash) { + public PackerImageParserBase(String bubbleVersion, String versionHash) { this.bubbleVersion = bubbleVersion; - this.keyHash = keyHash; + this.versionHash = versionHash; } public boolean isValidPackerImage(String name) { if (!name.startsWith(PACKER_IMAGE_PREFIX)) return false; if (!name.contains("_"+bubbleVersion+"_")) return false; - if (!name.contains("_"+keyHash+"_")) return false; + if (!name.contains("_"+versionHash+"_")) return false; return true; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java index 5b950d2f..b312bf61 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java @@ -235,7 +235,7 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { @Override public List getPackerImagesForRegion(String region) { return getPackerImages(); } public List getPackerImages () { - final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash())); + final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash())); return images == null ? Collections.emptyList() : images; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java index 3e13f559..b2b64c78 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java @@ -183,7 +183,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { final ArrayList filters = new ArrayList<>(); filters.add(new Filter("root-device-type", new SingletonList<>("ebs"))); filters.add(new Filter("state", new SingletonList<>("available"))); - filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerPublicKeyHash()+"_"+configuration.getShortVersion()+"_*"))); + filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerVersionHash()+"_"+configuration.getShortVersion()+"_*"))); final AmazonEC2 ec2 = getEc2Client(region); final DescribeImagesRequest imageRequest = new DescribeImagesRequest().withFilters(filters); final DescribeImagesResult imagesResult = ec2.describeImages(imageRequest); diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java index 2f498783..5165d39a 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java @@ -410,7 +410,7 @@ public class VultrDriver extends ComputeServiceDriverBase { @Override public List getPackerImagesForRegion(String region) { return getPackerImages(); } public List getPackerImages () { - final List images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash())); + final List images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash())); return images == null ? Collections.emptyList() : images; } @@ -424,16 +424,16 @@ public class VultrDriver extends ComputeServiceDriverBase { } // wait longer for the snapshot... - final String keyHash = packerService.getPackerPublicKeyHash(); + final String versionHash = packerService.getPackerVersionHash(); final long start = now(); PackerImage snapshot = null; while (now() - start < SNAPSHOT_TIMEOUT) { snapshot = getPackerImages().stream() - .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(keyHash)) + .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(versionHash)) .findFirst() .orElse(null); if (snapshot != null) break; - sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+keyHash); + sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+versionHash); } if (snapshot == null) { log.error("finalizeIncompletePackerRun: timeout waiting for snapshot"); @@ -447,14 +447,14 @@ public class VultrDriver extends ComputeServiceDriverBase { public boolean stopImageServer(AnsibleInstallType installType) { - final String keyHash = packerService.getPackerPublicKeyHash(); + final String versionHash = packerService.getPackerVersionHash(); final List servers; // find the server(s) try { servers = listNodes(server -> { final String tag = server.has(VULTR_TAG) ? server.get(VULTR_TAG).textValue() : null; - return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(keyHash); + return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(versionHash); }); } catch (IOException e) { log.error("stopImageServer: error listing servers: "+shortError(e), e); diff --git a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java index 49b7f969..bdc2027d 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java @@ -67,6 +67,14 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { return super.preCreate(matcher); } + @Override public AppMatcher postCreate(AppMatcher matcher, Object context) { + final BubbleApp app = appDAO.findByUuid(matcher.getApp()); + if (app == null) return die("postCreate("+ matcher.getUuid()+"): app not found: "+ matcher.getApp()); + ruleEngineService.flushCaches(); + primerService.prime(app); + return super.postCreate(matcher, context); + } + @Override public AppMatcher postUpdate(AppMatcher matcher, Object context) { final BubbleApp app = appDAO.findByUuid(matcher.getApp()); if (app == null) return die("postUpdate("+ matcher.getUuid()+"): app not found: "+ matcher.getApp()); diff --git a/bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java b/bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java index 807cb061..826de316 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java +++ b/bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java @@ -10,6 +10,8 @@ import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; +import java.util.Properties; + import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.wizard.model.SemanticVersion.isNewerVersion; @@ -19,6 +21,7 @@ public class BubbleVersionInfo { @Getter @Setter private String version; @Getter @Setter private String shortVersion; @Getter @Setter private String sha256; + @Getter @Setter private Properties software; public boolean valid() { return !empty(version) && !empty(sha256); } diff --git a/bubble-server/src/main/java/bubble/resources/cloud/NetworksResource.java b/bubble-server/src/main/java/bubble/resources/cloud/NetworksResource.java index 50fef760..83017381 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/NetworksResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/NetworksResource.java @@ -217,16 +217,19 @@ public class NetworksResource extends AccountOwnedResource { + final String url = followSpec.get("url").textValue(); + final String regex = followSpec.get("regex").textValue(); + final Integer group = followSpec.has("group") ? followSpec.get("group").asInt() : null; + final List headers = new ArrayList<>(); + for (String name : req.getHeaderNames()) { + final String value = req.getHeader(name); + headers.add(new NameAndValue(name, value)); + } + final List matches = applyRegexToUrl(url, headers, regex, group); + return matches == null ? null : StringUtil.toString(matches, "\n"); + })); + + } else if (followSpec.isTextual()) { + // just a regular follow -- chase redirects + return ok(redirectCache.computeIfAbsent(followSpec.textValue(), HttpUtil::chaseRedirects)); + } else { + final String json = json(followSpec); + log.error("followLink: invalid json (expected String or {regex, url}): "+json); + return notFound(json); + } } @Path(EP_ASSETS+"/{requestId}/{appId}") diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index 78dfc441..6724d59d 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -52,7 +52,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import java.beans.Transient; -import java.io.File; +import java.io.*; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -98,6 +98,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration public static final String TAG_SECURITY_LEVELS = "securityLevels"; public static final String TAG_RESTORE_MODE = "awaitingRestore"; public static final String TAG_RESTORING_IN_PROGRESS = "restoreInProgress"; + public static final String TAG_FULL_VERSION = "fullVersion"; public static final String TAG_JAR_VERSION = "jarVersion"; public static final String TAG_JAR_UPGRADE_AVAILABLE = "jarUpgradeAvailable"; public static final String TAG_MAX_USERS = "maxUsers"; @@ -166,6 +167,9 @@ public class BubbleConfiguration extends PgRestServerConfiguration @Getter @Setter private String letsencryptEmail; + @Getter @Setter private String releaseUrlBase; + @Getter(lazy=true) private final SoftwareVersions softwareVersions = new SoftwareVersions(getReleaseUrlBase()); + @Setter private String localStorageDir = DEFAULT_LOCAL_STORAGE_DIR; public String getLocalStorageDir () { return empty(localStorageDir) ? DEFAULT_LOCAL_STORAGE_DIR : localStorageDir; } @@ -246,7 +250,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration return SCHEME_HTTPS + node.getFqdn() + ":" + node.getSslPort() + getHttp().getBaseUri(); } - private String getVersion () { + public static String getVersion () { final Properties properties = new Properties(); try { properties.load(loadResourceAsStream("META-INF/bubble/bubble.properties")); @@ -264,7 +268,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration return new BubbleVersionInfo() .setVersion(version) .setShortVersion(shortVersion) - .setSha256(getJarSha()); + .setSha256(getJarSha()) + .setSoftware(getSoftwareVersions().getDefaultSoftwareVersions()); } public String getShortVersion () { return getVersionInfo().getShortVersion(); } @@ -366,6 +371,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration {TAG_SSL_PORT, getDefaultSslPort()}, {TAG_SUPPORT, getSupport()}, {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()}, + {TAG_FULL_VERSION, getVersionInfo()}, {TAG_JAR_VERSION, getVersion()}, {TAG_JAR_UPGRADE_AVAILABLE, getJarUpgradeAvailable() ? getSageVersion() : null}, {TAG_MAX_USERS, plan == null ? null : plan.getMaxAccounts()}, diff --git a/bubble-server/src/main/java/bubble/server/BubbleServer.java b/bubble-server/src/main/java/bubble/server/BubbleServer.java index a59a70ae..47d8f311 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleServer.java +++ b/bubble-server/src/main/java/bubble/server/BubbleServer.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicReference; import static bubble.ApiConstants.HOME_DIR; import static bubble.model.cloud.BubbleNode.nodeFromFile; +import static bubble.server.BubbleConfiguration.getVersion; import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -86,14 +87,14 @@ public class BubbleServer extends RestServerBase { restoreKey.set(getRestoreKey()); if (restoreKey.get() != null) { - log.info("Starting BubbleServer in restore mode..."); + log.info("Starting BubbleServer ("+getVersion()+") in restore mode..."); main(BubbleServer.class, RESTORE_LIFECYCLE_LISTENERS, configSource, env); } else { if (Boolean.parseBoolean(env.get(BUBBLE_DUMP_CONFIG))) { log.info("Dumping BubbleConfiguration..."); dumpConfig(configSource); } else { - log.info("Starting BubbleServer..."); + log.info("Starting BubbleServer ("+getVersion()+")..."); main(BubbleServer.class, LIFECYCLE_LISTENERS, configSource, env); } } diff --git a/bubble-server/src/main/java/bubble/server/SoftwareVersions.java b/bubble-server/src/main/java/bubble/server/SoftwareVersions.java new file mode 100644 index 00000000..8af71f2d --- /dev/null +++ b/bubble-server/src/main/java/bubble/server/SoftwareVersions.java @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.server; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.FileUtil; + +import java.io.*; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static bubble.ApiConstants.HOME_DIR; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.http.HttpUtil.url2string; + +@Slf4j +public class SoftwareVersions { + + public static final String ROLE_ALGO = "algo"; + public static final String ROLE_MITMPROXY = "mitmproxy"; + public static final String ROLE_DNSCRYPT = "dnscrypt-proxy"; + public static final String ROLE_BUBBLE = "bubble"; + public static final String[] VERSIONED_SOFTWARE = {ROLE_DNSCRYPT, ROLE_ALGO, ROLE_MITMPROXY}; + + public static final File SOFTWARE_VERSIONS_FILE = new File(HOME_DIR+"/bubble_versions.properties"); + + public static final String SUFFIX_VERSION = "_version"; + public static final String SUFFIX_SHA = "_sha"; + + private final String releaseUrlBase; + + public SoftwareVersions (String releaseUrlBase) { this.releaseUrlBase = releaseUrlBase; } + + public String getRolePropBase(String roleName) { return roleName.replace("-", "_"); } + + public String getLatestVersion(String r) { + try { + return url2string(releaseUrlBase+"/"+ r +"/latest.txt").trim(); + } catch (IOException e) { + return die("getLatestVersion("+ r +"): "+shortError(e), e); + } + } + + public String downloadHash(String roleName, String version) { + try { + return url2string(releaseUrlBase+"/"+ roleName +"/"+ version +"/"+ roleName +getSoftwareSuffix(roleName)+".sha256").trim(); + } catch (IOException e) { + return die("getSoftwareHash("+ roleName +"): "+shortError(e), e); + } + } + + @Getter(lazy=true) private final Properties defaultSoftwareVersions = initDefaultSoftwareVersions(); + private Properties initDefaultSoftwareVersions() { + if (empty(releaseUrlBase)) { + log.warn("initDefaultSoftwareVersions: releaseUrlBase not defined"); + return null; + } + final Properties props = new Properties(); + if (!SOFTWARE_VERSIONS_FILE.exists()) { + // write latest versions + for (String roleName : VERSIONED_SOFTWARE) { + final String latestVersion = getLatestVersion(roleName); + props.setProperty(getRolePropBase(roleName)+SUFFIX_VERSION, latestVersion); + props.setProperty(getRolePropBase(roleName)+SUFFIX_SHA, downloadHash(roleName, latestVersion)); + } + writeVersions(props, SOFTWARE_VERSIONS_FILE); + } + try (InputStream in = new FileInputStream(SOFTWARE_VERSIONS_FILE)) { + props.load(in); + return props; + } catch (Exception e) { + log.error("initDefaultSoftwareVersions: "+shortError(e)); + return null; + } + } + + public void writeVersions(File file) { writeVersions(getDefaultSoftwareVersions(), file); } + + public void writeVersions(Properties props, File file) { + try (OutputStream out = new FileOutputStream(file)) { + props.store(out, null); + } catch (Exception e) { + log.error("saveSoftwareVersions: "+shortError(e)); + } + } + + public void writeAnsibleVars(File file) { writeAnsibleVars(getDefaultSoftwareVersions(), file); } + + public void writeAnsibleVars(Properties props, File file) { + try (OutputStream out = new FileOutputStream(file)) { + final StringBuilder b = new StringBuilder(); + for (String name : props.stringPropertyNames()) { + b.append(name).append(" : '").append(props.getProperty(name)).append("'\n"); + } + FileUtil.toFile(file, b.toString()); + + } catch (Exception e) { + die("writeAnsibleVars: "+shortError(e)); + } + } + + private final Map softwareVersions = new HashMap<>(); + + public String getSoftwareVersion(String roleName) { + final Properties defaults = getDefaultSoftwareVersions(); + if (defaults != null) { + final String propName = getRolePropBase(roleName) + SUFFIX_VERSION; + final String version = defaults.getProperty(propName); + if (version != null) return version; + } + return softwareVersions.computeIfAbsent(roleName, this::getLatestVersion); + } + + private final Map softwareHashes = new HashMap<>(); + + public String getSoftwareHash(String roleName, String version) { + final Properties defaults = getDefaultSoftwareVersions(); + if (defaults != null) { + final String roleBase = getRolePropBase(roleName); + final String foundVersion = defaults.getProperty(roleBase + SUFFIX_VERSION); + if (foundVersion != null && foundVersion.equals(version)) { + final String hash = defaults.getProperty(roleBase + SUFFIX_SHA); + if (hash != null) return hash; + } + } + return softwareHashes.computeIfAbsent(roleName, r -> downloadHash(r, version)); + } + + private String getSoftwareSuffix(String roleName) { + switch (roleName) { + case ROLE_ALGO: case ROLE_MITMPROXY: return ".zip"; + case ROLE_DNSCRYPT: return ""; + default: return die("getSoftwareSuffix: unrecognized roleName: "+roleName); + } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java b/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java index 4e46511e..85e7e1ad 100644 --- a/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java +++ b/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java @@ -441,14 +441,12 @@ public class StandardSelfNodeService implements SelfNodeService { return planDAO.findByUuid(accountPlan.getPlan()); } - @Override - public boolean getLogFlag() { + @Override public boolean getLogFlag() { var flagStr = getNodeConfig().get_plaintext(REDIS_LOG_FLAG_KEY); - return empty(flagStr) ? false : Boolean.valueOf(flagStr); + return Boolean.parseBoolean(flagStr); } - @Override - @NonNull public Optional getLogFlagExpirationTime() { + @Override @NonNull public Optional getLogFlagExpirationTime() { var ttl = getNodeConfig().get_ttl(REDIS_LOG_FLAG_KEY); return ttl < 0 ? Optional.empty() : Optional.of(now() + ttl * 1000); } diff --git a/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java b/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java index 245d9825..e26ff231 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java @@ -161,7 +161,7 @@ public class AnsiblePrepService { if (installType == AnsibleInstallType.sage) return (int) (((double) memoryMB) * 0.6d); if (memoryMB >= 4096) return (int) (((double) memoryMB) * 0.6d); if (memoryMB >= 2048) return (int) (((double) memoryMB) * 0.5d); - if (memoryMB >= 1024) return (int) (((double) memoryMB) * 0.196d); + if (memoryMB >= 1024) return (int) (((double) memoryMB) * 0.24d); // no nodes are this small, API probably would not start, not enough memory return (int) (((double) memoryMB) * 0.19d); } diff --git a/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java b/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java index a0f69c4e..4ec8c95d 100644 --- a/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java +++ b/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java @@ -42,6 +42,7 @@ import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.wizard.server.config.PgRestServerConfiguration.ENV_PGPASSWORD; +import static org.cobbzilla.wizard.server.listener.FlywayMigrationListener.getFlywayTableName; @Service @Slf4j public class DatabaseFilterService { @@ -150,14 +151,19 @@ public class DatabaseFilterService { return die("copyDatabase: writer exited with an error (dbName="+dbName+"): "+writeResult.get()); } - // copy flyway schema table - log.debug("copyDatabase: dumping flyway_schema_version data"); - final CommandResult flywayData = pgExec("pg_dump", dbConfig.getDatabaseName(), null, null, FLYWAY_DUMP_OPTIONS); - if (!flywayData.isZeroExitStatus()) return die("copyDatabase: error dumping flyway_schema_version data: "+flywayData); - - log.debug("copyDatabase: inserting flyway_schema_version data"); - final CommandResult flywayInsert = pgExec("psql", dbName, new ByteArrayInputStream(flywayData.getStdout().getBytes()), null); - if (!flywayInsert.isZeroExitStatus()) return die("copyDatabase: error inserting flyway_schema_version data: "+flywayInsert); + // copy flyway schema table, if it exists. + // it may not exist if this is the very first time the server has been run + if (configuration.tableExists(getFlywayTableName())) { + log.debug("copyDatabase: dumping flyway_schema_version data"); + final CommandResult flywayData = pgExec("pg_dump", dbConfig.getDatabaseName(), null, null, FLYWAY_DUMP_OPTIONS); + if (!flywayData.isZeroExitStatus()) return die("copyDatabase: error dumping flyway_schema_version data: " + flywayData); + + log.debug("copyDatabase: inserting flyway_schema_version data"); + final CommandResult flywayInsert = pgExec("psql", dbName, new ByteArrayInputStream(flywayData.getStdout().getBytes()), null); + if (!flywayInsert.isZeroExitStatus()) return die("copyDatabase: error inserting flyway_schema_version data: "+flywayInsert); + } else { + log.warn("copyDatabase: flyway table ("+getFlywayTableName()+") does not exist, not copying"); + } // dump new DB log.info("copyDatabase: dumping new database: "+dbName); diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index addf2353..559da568 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -16,6 +16,7 @@ import bubble.model.account.Account; import bubble.model.cloud.AnsibleInstallType; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; +import bubble.server.SoftwareVersions; import bubble.service.cloud.GeoService; import lombok.Cleanup; import lombok.Getter; @@ -33,6 +34,7 @@ import org.cobbzilla.util.time.TimeUtil; import org.springframework.beans.factory.annotation.Autowired; import java.io.File; +import java.io.IOException; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicReference; @@ -40,6 +42,7 @@ import java.util.stream.Collectors; import static bubble.ApiConstants.copyScripts; import static bubble.model.cloud.RegionalServiceDriver.findClosestRegions; +import static bubble.server.SoftwareVersions.*; import static bubble.service.packer.PackerService.*; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.io.FileUtil.*; @@ -61,12 +64,12 @@ public class PackerJob implements Callable> { public static final String INSTALL_TYPE_VAR = "@@TYPE@@"; public static final String SAGE_NET_VAR = "@@SAGE_NET@@"; - public static final String PACKER_KEY_VAR = "@@PACKER_KEY_HASH@@"; + public static final String PACKER_VERSION_HASH_VAR = "@@PACKER_VERSION_HASH@@"; public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@"; public static final String TIMESTAMP_VAR = "@@TIMESTAMP@@"; public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX + INSTALL_TYPE_VAR + "_" + SAGE_NET_VAR - + "_" + PACKER_KEY_VAR + + "_" + PACKER_VERSION_HASH_VAR + "_" + BUBBLE_VERSION_VAR + "_" + TIMESTAMP_VAR; @@ -157,6 +160,11 @@ public class PackerJob implements Callable> { // copy ansible and other packer files to temp dir @Cleanup final TempDir tempDir = copyClasspathDirectory("packer"); + // record versions of algo, mitmproxy and dnscrypt_proxy + writeSoftwareVars(ROLE_ALGO, tempDir); + writeSoftwareVars(ROLE_MITMPROXY, tempDir); + writeSoftwareVars(ROLE_BUBBLE, tempDir); + // copy packer ssh key copyFile(packerService.getPackerPublicKey(), new File(abs(tempDir)+"/roles/common/files/"+PACKER_KEY_NAME)); @@ -212,7 +220,7 @@ public class PackerJob implements Callable> { final String imageName = PACKER_IMAGE_NAME_TEMPLATE .replace(INSTALL_TYPE_VAR, installType.name()) .replace(SAGE_NET_VAR, truncate(domainname(), 19)) - .replace(PACKER_KEY_VAR, packerService.getPackerPublicKeyHash()) + .replace(PACKER_VERSION_HASH_VAR, packerService.getPackerVersionHash()) .replace(BUBBLE_VERSION_VAR, configuration.getShortVersion()) .replace(TIMESTAMP_VAR, TimeUtil.format(now(), DATE_FORMAT_YYYYMMDDHHMMSS)); if (imageName.length() > 128) return die("imageName.length > 128: "+imageName); // sanity check @@ -282,6 +290,12 @@ public class PackerJob implements Callable> { return images; } + private void writeSoftwareVars(String roleName, TempDir tempDir) throws IOException { + final SoftwareVersions softwareVersions = configuration.getSoftwareVersions(); + final File varsDir = mkdirOrDie(abs(tempDir) + "/roles/"+roleName+"/vars"); + softwareVersions.writeAnsibleVars(new File(varsDir, "main.yml")); + } + private List getRolesForInstallType(AnsibleInstallType installType) { switch (installType) { case sage: return SAGE_ROLES; diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerService.java b/bubble-server/src/main/java/bubble/service/packer/PackerService.java index 4272ce19..5962aa03 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerService.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerService.java @@ -8,9 +8,9 @@ import bubble.cloud.compute.PackerImage; import bubble.model.cloud.AnsibleInstallType; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; +import bubble.server.SoftwareVersions; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.daemon.DaemonThreadFactory; -import org.cobbzilla.util.security.ShaUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -22,10 +22,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static bubble.server.SoftwareVersions.*; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.io.FileUtil.mkdirOrDie; import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.security.ShaUtil.sha256_file; import static org.cobbzilla.util.string.StringUtil.splitAndTrim; import static org.cobbzilla.util.system.CommandShell.chmod; import static org.cobbzilla.util.system.CommandShell.execScript; @@ -40,6 +42,7 @@ public class PackerService { public static final List NODE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/node-roles.txt"), "\n") .stream().filter(s -> !empty(s)).collect(Collectors.toList()); + public static final String PACKER_KEY_NAME = "packer_rsa"; private final Map activeJobs = new ConcurrentHashMap<>(16); @@ -82,7 +85,18 @@ public class PackerService { public File getPackerPublicKey () { return initPackerKey(true); } public File getPackerPrivateKey () { return initPackerKey(false); } - public String getPackerPublicKeyHash () { return ShaUtil.sha256_file(getPackerPublicKey()); } + public String getPackerPublicKeyHash () { return sha256_file(getPackerPublicKey()); } + + public String getPackerVersionHash () { + final SoftwareVersions softwareVersions = configuration.getSoftwareVersions(); + final String keyHash = getPackerPublicKeyHash(); + final String versions = "" + +"_d"+softwareVersions.getSoftwareVersion(ROLE_DNSCRYPT) + +"_a"+softwareVersions.getSoftwareVersion(ROLE_ALGO) + +"_m"+softwareVersions.getSoftwareVersion(ROLE_MITMPROXY); + if (versions.length() > 48) return die("getPackerVersionHash: software versions are too long (versions.length == "+versions.length()+" > 48): "+versions); + return keyHash.substring(64 - versions.length())+versions; + } public synchronized File initPackerKey(boolean pub) { final File keyDir = new File(System.getProperty("user.home"),".ssh"); diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index a17f3fe7..b9f64a3e 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -214,6 +214,8 @@ public class StandardAppPrimerService implements AppPrimerService { } } catch (Exception e) { die("_prime: "+shortError(e), e); + } finally { + log.info("_primeApps: completed"); } } diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index a09b8007..443625cc 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.1.1 +bubble.version=Adventure 1.1.3 diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 2248261e..1a354f8b 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -19,4 +19,5 @@ install_packer.sh rkeys rmembers rdelkeys -mitm_pid \ No newline at end of file +mitm_pid +reset_bubble_logs \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json index 2059e558..40a2f9c8 100644 --- a/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json +++ b/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json @@ -6,6 +6,7 @@ {"name": "restore_key", "value": "[[restoreKey]]"}, {"name": "install_type", "value": "[[installType]]"}, {"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[jvmMaxRamMB]]m"}, + {"name": "total_memory", "value": "[[nodeSize.memoryMB]]"}, {"name": "cert_name", "value": "bubble-[[network.shortId]]"} ], "optionalConfigNames": ["restore_key"] diff --git a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml index 8e8479d1..57a82eb1 100644 --- a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml @@ -25,6 +25,11 @@ src: "supervisor_bubble.conf.j2" dest: /etc/supervisor/conf.d/bubble.conf +# Save 1% of memory, every bit counts on small nodes +- name: Disable peer manager on small nodes + shell: bash -c "supervisorctl stop bubble_peer_manager && rm /etc/supervisor/conf.d/bubble_peer_manager.conf" + when: total_memory < 2048 + - name: save iptables v4 rules shell: iptables-save > /etc/iptables/rules.v4 become: yes diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitm_monitor.conf b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitm_monitor.conf index ffae67ae..56e6ed2b 100644 --- a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitm_monitor.conf +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitm_monitor.conf @@ -3,3 +3,4 @@ stdout_logfile = /dev/null stderr_logfile = /dev/null command=/usr/local/sbin/mitm_monitor.sh +stopsignal=QUIT diff --git a/bubble-server/src/main/resources/bubble-config.yml b/bubble-server/src/main/resources/bubble-config.yml index 881bf568..dafdec99 100644 --- a/bubble-server/src/main/resources/bubble-config.yml +++ b/bubble-server/src/main/resources/bubble-config.yml @@ -76,6 +76,7 @@ localNotificationStrategy: {{#exists BUBBLE_LOCAL_NOTIFY}}{{BUBBLE_LOCAL_NOTIFY} letsencryptEmail: {{LETSENCRYPT_EMAIL}} localStorageDir: {{LOCALSTORAGE_BASE_DIR}} +releaseUrlBase: {{#exists BUBBLE_RELEASE_URL_BASE}}{{BUBBLE_RELEASE_URL_BASE}}{{else}}https://jenkins.bubblev.org/public/releases{{/exists}} disallowedCountries: {{DISALLOWED_COUNTRIES}} diff --git a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs index 437cf0cb..52706e65 100644 --- a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs @@ -87,10 +87,10 @@ function {{JS_PREFIX}}_create_button(labelKey, labelDefault, onclick, labelForma return btn; } -let {{PAGE_PREFIX}}_url_chasers = {}; - if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { + let {{PAGE_PREFIX}}_url_chasers = {}; + {{PAGE_PREFIX}}_screenWidth = function () { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth }; let {{PAGE_PREFIX}}_doc_ready = false; diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs new file mode 100644 index 00000000..d38f958a --- /dev/null +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs @@ -0,0 +1,102 @@ +{{JS_PREFIX}}_supports_keywords = true; + +const {{JS_PREFIX}}_site_host = location.protocol + '//' + window.location.hostname + '/'; + +function {{JS_PREFIX}}_apply_blocks(blocked_users) { + const articles = Array.from(document.getElementsByClassName('feed-shared-update-v2')); + if (articles === null || articles.length === 0) { + console.warn('No articles found, not filtering'); + return; + } + {{JS_PREFIX}}_consider_block(articles, blocked_users); +} + +function {{JS_PREFIX}}_author_from_href(href) { + if (typeof href === 'undefined' || href === null) return null; + let h = href.startsWith({{JS_PREFIX}}_site_host) ? href.substring({{JS_PREFIX}}_site_host.length) : href; + const qPos = h.indexOf('?'); + if (qPos !== -1) { + h = h.substring(0, qPos); + } + if (h.endsWith('/')) h = h.substring(0, h.length - 1); + if (!h.startsWith('in/') && !h.startsWith('company/')) { + return null; + } + const slashPos = h.indexOf('/'); + const name = h.substring(slashPos); + if (name.length > 35 && name.indexOf('-') === -1 && name.indexOf('_') === -1) return null; + console.log("author_from_href: found "+name+' from '+href); + return name; +} + +function {{JS_PREFIX}}_remove_article_from_dom(article) { + // todo: does this work? + article.parentNode.removeChild(article); +} + +function {{JS_PREFIX}}_create_block_control(article, authorName, articleLink) { + let linkClass = articleLink.className; + if (linkClass && linkClass.indexOf('{{JS_PREFIX}}_link_decorated') !== -1) { + return null; + } else { + articleLink.className = articleLink.className ? articleLink.className + ' {{JS_PREFIX}}_link_decorated' : '{{JS_PREFIX}}_link_decorated'; + } + const imgHolder = {{JS_PREFIX}}_create_block_img(); + const blockSpan = document.createElement('span'); + const blockLink = document.createElement('a'); + blockLink.style.zIndex = '{{APP_CONTROLS_Z_INDEX}}' + blockLink.style.cursor = 'pointer'; + blockLink.addEventListener("click", function (e) { + {{JS_PREFIX}}_remove_article_from_dom(article, authorName); + {{JS_PREFIX}}_block_user(authorName); + e.stopPropagation(); + e.preventDefault(); + return false; + }); + blockLink.appendChild(imgHolder); + blockSpan.appendChild(document.createTextNode('\u00A0\u00A0')); + blockSpan.appendChild(blockLink); + blockSpan.id = 'blockSpan_'+{{JS_PREFIX}}_uuidv4(); + console.log('adding block control on '+authorName); + return blockSpan; +} + +function {{JS_PREFIX}}_consider_block(articles, blocked_users) { + if (articles && articles.length && articles.length > 0) { + for (let i=0; iPromoted", + "data": "true" + }] + } +}] diff --git a/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json new file mode 100644 index 00000000..fba42acd --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json @@ -0,0 +1,15 @@ +[{ + "name": "UserBlocker", + "children": { + "AppMatcher": [{ + "name": "LIMatcher", + "site": "LinkedIn", + "template": true, + "requestCheck": true, + "requestModifier": true, + "fqdn": "www.linkedin.com", + "urlRegex": "/feed/", + "rule": "li_user_blocker" + }] + } +}] diff --git a/bubble-server/src/main/resources/models/manifest-app-user-block.json b/bubble-server/src/main/resources/models/manifest-app-user-block.json index 49e14659..c5c3f70b 100644 --- a/bubble-server/src/main/resources/models/manifest-app-user-block.json +++ b/bubble-server/src/main/resources/models/manifest-app-user-block.json @@ -15,6 +15,9 @@ "apps/user_block/insta/bubbleApp_userBlock_insta", "apps/user_block/insta/bubbleApp_userBlock_insta_matchers", "apps/user_block/insta/bubbleApp_userBlock_insta_data", + "apps/user_block/li/bubbleApp_userBlock_li", + "apps/user_block/li/bubbleApp_userBlock_li_matchers", + "apps/user_block/li/bubbleApp_userBlock_li_data", "apps/user_block/reddit/bubbleApp_userBlock_reddit", "apps/user_block/reddit/bubbleApp_userBlock_reddit_matchers" ] \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index 948be59e..a8337eef 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -11,9 +11,9 @@ - name: Download algo dist file get_url: - url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip + url: https://jenkins.bubblev.org/public/releases/algo/{{ algo_version }}/algo.zip dest: /tmp/algo.zip - checksum: sha256:0c32e213ff4cfd807718cb276f6160d4b587779c94fba2c2102905b7303720f8 + checksum: sha256:{{ algo_sha }} - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh index 007789b1..770c465c 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh @@ -28,6 +28,7 @@ NEW_KEYS=$(mktemp /root/.ssh/authorized_keys.XXXXXXX) chmod 600 ${NEW_KEYS} || die "Error setting permissions on new authorized_keys file: ${NEW_KEYS}" KEY_COUNT=0 +ALL_KEY_COUNT=0 for key in $(echo "${CURRENT_KEYS_SQL}" | PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble -qt) ; do if [[ -z "$(echo "${key}" | tr -d [[:space:]])" ]] ; then continue @@ -40,14 +41,19 @@ for key in $(echo "${CURRENT_KEYS_SQL}" | PGPASSWORD="$(cat /home/bubble/.BUBBLE else log "Warning: NOT adding malformed key: $(echo "${KEY}" | tr -d '\n')" fi + ALL_KEY_COUNT=$(expr ${ALL_KEY_COUNT} + 1) done if [[ ${KEY_COUNT} -eq 0 ]] ; then # Sanity check that we can even talk to psql # We may be out of memory, in which case we do not want to erase existing installed keys - if [[ -z "$(echo 'SELECT count(*) FROM account_ssh_key' | PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble)" ]] ; then + CHECK_KEY_COUNT="$(echo 'SELECT count(*) FROM account_ssh_key' | PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble)" + if [[ -z "${CHECK_KEY_COUNT}" ]] ; then log "Warning: error calling psql, not installing/uninstalling any keys" exit 0 + elif [[ ${CHECK_KEY_COUNT} -ne ${ALL_KEY_COUNT} ]] ; then + log "Warning: error CHECK_KEY_COUNT (${CHECK_KEY_COUNT}) -ne ALL_KEY_COUNT (${ALL_KEY_COUNT}), not installing/uninstalling any keys" + exit 0 fi fi diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml index efffb14d..2a8235bb 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -132,3 +132,11 @@ special_time: "hourly" user: "root" job: "find /tmp ~bubble/tmp -mtime +1 -type f -delete && find /tmp ~bubble/tmp -mtime +1 -type d -empty -delete" + +- name: Record software versions + template: + src: bubble_versions.properties.j2 + dest: /home/bubble/bubble_versions.properties + owner: bubble + group: bubble + mode: 0400 diff --git a/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 b/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 new file mode 100644 index 00000000..01cf6525 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 @@ -0,0 +1,6 @@ +algo_version={{ algo_version }} +algo_sha={{ algo_sha }} +dnscrypt_proxy_version={{ dnscrypt_proxy_version }} +dnscrypt_proxy_sha={{ dnscrypt_proxy_sha }} +mitmproxy_version={{ mitmproxy_version }} +mitmproxy_sha={{ mitmproxy_sha }} diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py index fa9b9630..6d9ee403 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py @@ -1,3 +1,36 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Adapted from _client.py in the httpx project. The httpx license is reprinted here: +# +# Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/). +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# import functools import typing from types import TracebackType @@ -1348,6 +1381,10 @@ class AsyncClient(BaseClient): if allow_redirects: await response.aread() + # THIS IS THE ONLY THING THAT WAS CHANGED + # The line below is indented here, was not in original file + # When allow_redirects is False, we do not need to build a redirect request, which fails + # because when the request is streamed, we can't read it again here request = self._build_redirect_request(request, response) history = history + [response] diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py index d5cfdf37..b35f38e3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py @@ -1,3 +1,31 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Adapted from _events.py in the h11 project. The h11 license is reprinted here: +# +# The MIT License (MIT) +# +# Copyright (c) 2016 Nathaniel J. Smith and other contributors +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# # High level events that make up HTTP/1.1 conversations. Loosely inspired by # the corresponding events in hyper-h2: # @@ -220,6 +248,8 @@ class Response(_ResponseBase): """ def _validate(self): + # THIS IS THE ONLY THING THAT WAS CHANGED + # Certain websites return HTTP status 999 to mean 200 if self.status_code == 999: self.status_code = 200 if not (200 <= self.status_code < 600): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 10cdd108..e16d3ac1 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -436,8 +436,8 @@ def is_flex_domain(client_addr, server_addr, fqdns): bubble_log.debug('is_flex_domain: returning True for: '+fqdn+' (check='+check_fqdn+')') return True check_fqdn = check_fqdn[check_fqdn.index('.')+1:] - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn) + # if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('is_flex_domain: returning False for: '+fqdn) return False diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 2f3267c6..4b18af78 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -2,17 +2,16 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # import asyncio +import base64 import json import re import urllib import traceback from mitmproxy.net.http import Headers -from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody from bubble_config import bubble_port, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri -from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ - CTX_BUBBLE_FLEX, CTX_BUBBLE_SPECIAL, \ +from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_FLEX, \ status_reason, get_flow_ctx, add_flow_ctx, bubble_async, async_client, cleanup_async, \ is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ @@ -64,10 +63,10 @@ def ensure_bubble_csp(csp, req_id): break # if no nonce, then add our nonce if not found_nonce: - new_csp = add_csp_part(new_csp, " ".join(tokens) + " 'nonce="+req_id+"'") + new_csp = add_csp_part(new_csp, " ".join(tokens) + " 'nonce-"+base64.b64encode(bytes(req_id, 'utf-8')).decode()+"' ") else: # does not allow from self, so add self with our nonce - new_csp = add_csp_part(new_csp, tokens[0] + " 'self' 'nonce="+req_id+"'" + " ".join(tokens[1:])) + new_csp = add_csp_part(new_csp, tokens[0] + " 'self' 'nonce-"+base64.b64encode(bytes(req_id, 'utf-8')).decode()+"' " + " ".join(tokens[1:])) else: new_csp = add_csp_part(new_csp, part) return new_csp @@ -285,8 +284,6 @@ def responseheaders(flow): flex_flow = process_flex(flex_flow) else: flex_flow = None - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('responseheaders: flex_flow = '+repr(flex_flow)) bubble_filter_response(flow, flex_flow) @@ -302,7 +299,8 @@ def bubble_filter_response(flow, flex_flow): if is_bubble_health_check(path): health_check_response(flow) else: - bubble_log.info('bubble_filter_response: sending special bubble response for path: '+path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_response: sending special bubble response for path: '+path) special_bubble_response(flow) elif flex_flow and flex_flow.is_error(): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 9eb26ae2..9615a56f 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -30,9 +30,9 @@ import uuid from mitmproxy.net.http import headers as nheaders from bubble_api import bubble_matchers, bubble_activity_log, \ - HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ + CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ - is_bubble_special_path, is_bubble_health_check, \ + is_bubble_special_path, is_bubble_health_check, health_check_response, \ is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias from bubble_flex import new_flex_flow @@ -49,11 +49,9 @@ class Rerouter: if host is None: return None - is_health_check = is_bubble_health_check(flow.request.path) if is_bubble_special_path(flow.request.path): - if not is_health_check: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path) return None client_addr = str(flow.client_conn.address[0]) @@ -138,7 +136,6 @@ class Rerouter: port = int(m.group("port")) # Determine if this request should be filtered - is_health_check = False host = None path = flow.request.path if sni or host_header: @@ -151,14 +148,17 @@ class Rerouter: # If http, we validate client/server here if is_http: fqdns = [host] - if is_bubble_request(server_addr, fqdns): - is_health_check = path.startswith(HEALTH_CHECK_URI) - if not is_health_check: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path) - add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) - add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) - return None + if is_bubble_health_check(path): + # Health check + health_check_response(flow) + return None + + elif is_bubble_request(server_addr, fqdns): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path) + add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) + add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) + return None elif is_sage_request(server_addr, fqdns): if bubble_log.isEnabledFor(DEBUG): @@ -170,7 +170,7 @@ class Rerouter: elif is_not_from_vpn(client_addr): # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url) + bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url+' host='+host) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return None @@ -225,10 +225,9 @@ class Rerouter: bubble_log.debug('bubble_handle_request: no rules returned, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) else: - if not is_health_check: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...') - # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...') + # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) elif is_http and is_not_from_vpn(client_addr): # todo: add to fail2ban diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh index e5a6a1ea..06e6f08a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh @@ -3,6 +3,10 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # LOG=/var/log/bubble/mitm_monitor.log +if [[ ! -f ${LOG} ]] ; then + touch ${LOG} +fi +chgrp mitmproxy ${LOG} && chmod 660 ${LOG} # allow run_mitm.sh to write to this log function die { echo 1>&2 "${1}" @@ -76,22 +80,25 @@ function fullMitmReset { function healthCheck { MITM_PORT=${1} START=$(date +%s) - HEALTH_CHECK_TIMEOUT=20 + HEALTH_CHECK_TIMEOUT=30 HEALTH_OK="NOT_RUN" + HC_URL="http://$(hostname):${MITM_PORT}/__bubble/__mitm_health" while [[ $(expr $(date +%s) - ${START}) -le ${HEALTH_CHECK_TIMEOUT} ]] ; do - # log "Performing health check on mitm${MITM_PORT}..." - CURL_OUT="$(curl --silent --connect-timeout 2 --max-time 2 http://$(hostname):${MITM_PORT}/__bubble/__mitm_health 2>> ${LOG})" + # log "Performing health check on mitm${MITM_PORT} via ${HC_URL} ..." + CURL_OUT="$(curl --silent --connect-timeout 2 --max-time 2 ${HC_URL} 2>> ${LOG})" if [[ ! -z ${CURL_OUT} && ${CURL_OUT} == "OK" ]] ; then - # log "Health check on mitm${MITM_PORT}: OK" + # log "Health check on mitm${MITM_PORT} via ${HC_URL} : OK" echo -n "OK" return else - log "Health check on mitm${MITM_PORT}: failed: ${CURL_OUT}" + log "Health check on mitm${MITM_PORT} via ${HC_URL} failed: ${CURL_OUT}" HEALTH_OK="CURL_FAIL" fi - sleep 1s + sleep 5s + DELTA=$(expr $(date +%s) - ${START}) + log "$(expr ${HEALTH_CHECK_TIMEOUT} - ${DELTA}) seconds before restart" done - log "Health check: final failure for mitm${MITM_PORT}, returning ${HEALTH_OK}" + log "Health check: final failure for mitm${MITM_PORT} via ${HC_URL} returning ${HEALTH_OK}" echo -n "${HEALTH_OK}" } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh index 0bcd219a..36747c2a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh @@ -3,12 +3,40 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # PORT=${1:-8888} -echo "Starting mitmproxy on port ${PORT} ..." +MITM_PORT_FILE=/home/mitmproxy/mitmproxy_port +LOG=/var/log/bubble/mitm_monitor.log + +function log { + echo "[mitm${PORT}] $(date): ${1}" | tee -a ${LOG} +} + +if [[ -f ${MITM_PORT_FILE} ]] ; then + MITM_PORT="$(cat ${MITM_PORT_FILE})" + START=$(date +%s) + TIMEOUT=30 + while [[ ! -s "${MITM_PORT_FILE}" ]] ; do + log "MITM_PORT_FILE was empty: ${MITM_PORT_FILE} -- waiting for it to exist" + if [[ $(expr $(date +%s) - ${START}) -gt ${TIMEOUT} ]] ; then + log "timeout waiting for MITM_PORT_FILE to exist: ${MITM_PORT_FILE} -- starting anyway" + break + fi + sleep 5s + done + if [[ -s ${MITM_PORT_FILE} ]] ; then + MITM_PORT="$(cat ${MITM_PORT_FILE})" + if [[ ! -z "${MITM_PORT}" && ${MITM_PORT} -ne ${PORT} ]] ; then + log "Our port (${PORT}) is not the primary mitm port (${MITM_PORT}), delaying startup by 30 seconds" + sleep 30s + fi + fi +fi + +log "Starting mitmproxy on port ${PORT} ..." VENV_DIR="/home/mitmproxy/mitmproxy/venv" SETUP_VENV=1 if [[ -d ${VENV_DIR} && $(find ${VENV_DIR} -type f -name "redis*" | wc -c | tr -d ' ') -gt 0 ]] ; then - echo "venv dir looks OK, skipping venv setup" + log "venv dir looks OK, skipping venv setup" SETUP_VENV=0 fi diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index 172c6e86..7e74193c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -39,9 +39,9 @@ - name: Download mitmproxy dist file get_url: - url: https://github.com/getbubblenow/bubble-dist/raw/master/mitmproxy/mitmproxy.zip + url: https://jenkins.bubblev.org/public/releases/mitmproxy/{{ mitmproxy_version }}/mitmproxy.zip dest: /tmp/mitmproxy.zip - checksum: sha256:eebde85f9d49f34634695c3e3916f5940a31e5393d52d3f2df793a2a4f9f6430 + checksum: sha256:{{ mitmproxy_sha }} - name: Unzip mitmproxy.zip unarchive: @@ -93,14 +93,14 @@ shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'" - name: Patch _client.py from httpx to fix bug with HTTP/2 redirects - file: + copy: src: _client.py dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/httpx/_client.py owner: mitmproxy group: mitmproxy - name: Patch _events.py from h11 to fix bug with HTTP status 999 being considered invalid - file: + copy: src: _events.py dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/h11/_events.py owner: mitmproxy diff --git a/bubble-web b/bubble-web index 3a0cbf5a..84b746a3 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 3a0cbf5aee69d9a88b3a3b42a658038e9c3d4016 +Subproject commit 84b746a3751a68c71b3d60e6a3a30d08e93b8b82 diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index c66f9e16..ea72ac4a 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit c66f9e167ca32887374ec07a05fd3bddaeb8c2d5 +Subproject commit ea72ac4a1619c4f5915047650cdd18b8a6202681 diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 7716949a..cd9cf18c 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 7716949a32c967ae62989d52745146b9cfcb8698 +Subproject commit cd9cf18c90a373f024163c048d57cf574ec97293