diff --git a/bin/prep_bubble_jar b/bin/prep_bubble_jar index 67ba5e17..1bf7cbcb 100755 --- a/bin/prep_bubble_jar +++ b/bin/prep_bubble_jar @@ -5,12 +5,9 @@ # # Prepares the bubble.jar file for active usage. # -# 1. Update role JSON in bubble-server/src/main/resources/ansible/default_roles.json -# Inserts "tgzB64" value with file://path to tarball +# 1. Copy scripts to bubble-server/target/classes/scripts # -# 2. Copy scripts to bubble-server/target/classes/scripts -# -# 3. If the environment variable INSTALL_WEB is equal to "web", also build and install the bubble-web +# 2. If the environment variable INSTALL_WEB is equal to "web", also build and install the bubble-web # site to bubble-server/target/classes/site # # Usage: @@ -33,88 +30,17 @@ fi BUBBLE_SERVER="$(cd "${SCRIPT_DIR}/../bubble-server" && pwd)" CLASSES_DIR="${BUBBLE_SERVER}/target/classes" -DEFAULT_ROLES_RELATIVE="ansible/default_roles.json" -DEFAULT_ROLES="${CLASSES_DIR}/${DEFAULT_ROLES_RELATIVE}" -if [[ ! -f ${DEFAULT_ROLES} ]] ; then - die "default roles file not found: ${DEFAULT_ROLES}" -fi -if [[ -z "${LOCALSTORAGE_BASE_DIR}" ]] ; then - for f in "${HOME}/bubble/current/bubble.env" "${HOME}/.bubble.env" ; do - if [[ -f "${f}" ]] ; then - LOCALSTORAGE_BASE_DIR=$(cat ${f} | grep -v '^#' | grep LOCALSTORAGE_BASE_DIR | awk -F '=' '{print $2}' | tr -d ' ') - break - fi - done -fi -if [[ -z "${LOCALSTORAGE_BASE_DIR}" ]] ; then - echo "Warning: LOCALSTORAGE_BASE_DIR env var not defined and no bubble.env found, using ${HOME}/.bubble_local_storage" - LOCALSTORAGE_BASE_DIR="${HOME}/.bubble_local_storage" -fi if [[ -z "${BUBBLE_JAR}" ]] ; then die "bubble jar not found: ${BUBBLE_JAR}" fi -ROLES_DIR="$(cd "${SCRIPT_DIR}/../automation/roles" && pwd)" -if [[ ! -d ${ROLES_DIR} ]] ; then - die "automation/roles dir not found: ${ROLES_DIR}" -fi - -LOCAL_NET_ID="$("${SCRIPT_DIR}/bconst" bubble.ApiConstants.ROOT_NETWORK_UUID 2> /dev/null)" -if [[ -z "${LOCAL_NET_ID}" ]] ; then - # try to read from source file - LOCAL_NET_ID="$(cat "${BUBBLE_SERVER}/src/main/java/bubble/ApiConstants.java" | grep -v '//' | egrep '\s+String\s+ROOT_NETWORK_UUID' | awk -F '"' '{print $2}')" - if [[ -z "${LOCAL_NET_ID}" ]] ; then - die "ROOT_NETWORK_UUID could not be read from ApiConstants" - fi -fi - -echo "lbs = ${LOCALSTORAGE_BASE_DIR}" - -UPDATED="$(mktemp /tmp/default_roles.XXXXXXX.json)" -cd ${ROLES_DIR} -echo "[" > "${UPDATED}" -for role in $(ls -1) ; do - echo "Processing role: ${role}" - ROLE_JSON="${role}/files/bubble_role.json" - if [[ ! -f "${ROLE_JSON}" ]] ; then - die "Json file not found for role ${role}: ${ROLE_JSON}" - fi - if [[ $(cat ${UPDATED} | wc -c) -gt 2 ]] ; then - echo "," >> ${UPDATED} - fi - - role_name="$(cat "${ROLE_JSON}" | jq -r .name)" - role_path="automation/roles/${role_name}.tgz" - TGZ_PATH="${LOCALSTORAGE_BASE_DIR}/${role_path}" - mkdir -p $(dirname ${TGZ_PATH}) || die "Error creating parent dir for ${TGZ_PATH}" - tar czf ${TGZ_PATH} ${role} - - cat ${ROLE_JSON} | jq --arg tgzB64 "storage://LocalStorage/${role_path}" '. + {tgzB64: $tgzB64}' >> ${UPDATED} - - echo "------------------------------" - echo "Generated role JSON: ${role}" - echo "------------------------------" -done - -echo "]" >> ${UPDATED} - -jq . < ${UPDATED} > ${DEFAULT_ROLES} || die "Error writing ${DEFAULT_ROLES}, maybe some problems with ${UPDATED} ?" - -echo "------------------------------------------------------------" -cat "${UPDATED}" -echo "------------------------------------------------------------" - -cd ${LOCALSTORAGE_BASE_DIR} && jar uvf ${BUBBLE_JAR} automation || die "Error updating ${BUBBLE_JAR} with default role archives" -mkdir -p ${LOCALSTORAGE_BASE_DIR}/${LOCAL_NET_ID} && cp -R ${LOCALSTORAGE_BASE_DIR}/automation ${LOCALSTORAGE_BASE_DIR}/${LOCAL_NET_ID}/ || die "Error creating/copying network storage dir: ${LOCALSTORAGE_BASE_DIR}/${LOCAL_NET_ID}" - mkdir -p ${CLASSES_DIR}/scripts for script in $(cat ${BUBBLE_SERVER}/src/main/resources/ansible/bubble_scripts.txt) ; do cp ${SCRIPT_DIR}/${script} ${CLASSES_DIR}/scripts || die "Error copying ${SCRIPT_DIR}/${script} -> ${CLASSES_DIR}/scripts" done -cd ${CLASSES_DIR} && jar uvf ${BUBBLE_JAR} scripts ${DEFAULT_ROLES_RELATIVE} || die "Error updating ${BUBBLE_JAR} with scripts" -echo "Updated $(ls -1 ${ROLES_DIR} | wc -l) roles in ${DEFAULT_ROLES}" +cd ${CLASSES_DIR} && jar uvf ${BUBBLE_JAR} scripts || die "Error updating ${BUBBLE_JAR} with scripts" rm -f "${UPDATED}" diff --git a/bin/run.sh b/bin/run.sh index 4db277be..3c47fb79 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -101,7 +101,7 @@ else fi if [[ -z "${BUBBLE_JAR}" ]] ; then - die "API jar file not found in ${BASE}/target" + die "API jar file not found" fi if [[ -z "${BUBBLE_JVM_OPTS}" ]] ; then diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index bc33f25e..2fb40c6a 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -41,6 +41,9 @@ public class ApiConstants { public static final String DEFAULT_LOCALE = "en_US"; + public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"}; + public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"}; + private static final AtomicReference bubbleDefaultDomain = new AtomicReference<>(); public static final ObjectMapper DB_JSON_MAPPER = COMPACT_MAPPER; diff --git a/bubble-server/src/main/java/bubble/cloud/compute/AnsibleRole.java b/bubble-server/src/main/java/bubble/cloud/compute/AnsibleRole.java new file mode 100644 index 00000000..3bbf1357 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/AnsibleRole.java @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.cloud.compute; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.wizard.model.entityconfig.annotations.ECField; +import org.cobbzilla.wizard.model.entityconfig.annotations.ECSearchable; + +import javax.persistence.Column; +import javax.persistence.Transient; +import javax.validation.constraints.Size; +import java.util.Arrays; + +import static bubble.ApiConstants.DB_JSON_MAPPER; +import static org.cobbzilla.util.json.JsonUtil.json; + +@NoArgsConstructor @Accessors(chain=true) +public class AnsibleRole { + + @Getter @Setter private String name; + + @ECSearchable(filter=true) @ECField(index=30) + @Size(max=10000, message="err.description.length") + @Getter @Setter private String description; + + @Column(updatable=false, length=10000) + @JsonIgnore @Getter @Setter private String configJson; + public boolean hasConfig () { return configJson != null; } + + @Transient public NameAndValue[] getConfig () { return configJson == null ? null : json(configJson, NameAndValue[].class); } + public AnsibleRole setConfig(NameAndValue[] config) { return setConfigJson(config == null ? null : json(config, DB_JSON_MAPPER)); } + + @Column(updatable=false, length=1000) @ECField(index=80) + @JsonIgnore @Getter @Setter private String optionalConfigNamesJson; + public boolean hasOptionalConfigNames () { return optionalConfigNamesJson != null; } + + @Transient public String[] getOptionalConfigNames() { return optionalConfigNamesJson == null ? null : json(optionalConfigNamesJson, String[].class); } + public AnsibleRole setOptionalConfigNames(String[] names) { return setOptionalConfigNamesJson(name == null ? null : json(names, DB_JSON_MAPPER)); } + + public boolean isOptionalConfigName(String cfgName) { + final String[] names = getOptionalConfigNames(); + if (names == null) return false; + return Arrays.asList(names).contains(cfgName); + } +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java index 64280f75..224e34b2 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java @@ -14,6 +14,7 @@ public class ComputeConfig extends RegionalConfig { @Getter @Setter private ComputeNodeSize[] sizes; @Getter @Setter private NameAndValue[] config; + @Getter @Setter private PackerConfig packer; public CloudRegion getRegion (String name) { for (CloudRegion r : getRegions()) { diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java new file mode 100644 index 00000000..cc230da2 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java @@ -0,0 +1,13 @@ +package bubble.cloud.compute; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.collection.NameAndValue; + +public class PackerConfig { + + @Getter @Setter private NameAndValue[] vars; + @Getter @Setter private JsonNode builder; + +} diff --git a/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java b/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java index 6158426f..205f2f5a 100644 --- a/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java @@ -27,7 +27,6 @@ import java.util.Arrays; import java.util.List; import static bubble.ApiConstants.ROOT_NETWORK_UUID; -import static bubble.dao.cloud.AnsibleRoleDAO.ROLE_PATH; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.json.JsonUtil.json; @@ -77,9 +76,6 @@ public class LocalStorageDriver extends CloudServiceDriverBase implements SqlViewSearc @Autowired private AppDataDAO dataDAO; @Autowired private AppMessageDAO appMessageDAO; @Autowired private RuleDriverDAO driverDAO; - @Autowired private AnsibleRoleDAO roleDAO; @Autowired private BubbleDomainDAO domainDAO; @Autowired private CloudServiceDAO cloudDAO; @Autowired private BubbleFootprintDAO footprintDAO; @@ -231,7 +229,6 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc ready.set(true); cloudDAO.ensureNoopCloudsExist(account); - copyTemplateObjects(acct, parent, roleDAO); final Map drivers = new HashMap<>(); copyTemplateObjects(acct, parent, driverDAO, new AccountTemplate.CopyTemplate<>() { @@ -326,7 +323,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc } @Transactional(Transactional.TxType.REQUIRES_NEW) - private void deleteTransactional(@NonNull final String uuid) { + protected void deleteTransactional(@NonNull final String uuid) { // loading, and actually checking if the account with given UUID exists final var account = findByUuid(uuid); diff --git a/bubble-server/src/main/java/bubble/dao/cloud/AnsibleRoleDAO.java b/bubble-server/src/main/java/bubble/dao/cloud/AnsibleRoleDAO.java deleted file mode 100644 index 5c6830a9..00000000 --- a/bubble-server/src/main/java/bubble/dao/cloud/AnsibleRoleDAO.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.dao.cloud; - -import bubble.cloud.CloudServiceType; -import bubble.dao.account.AccountOwnedTemplateDAO; -import bubble.model.account.Account; -import bubble.model.cloud.AnsibleRole; -import bubble.model.cloud.CloudService; -import bubble.server.BubbleConfiguration; -import bubble.service.cloud.StorageService; -import lombok.Cleanup; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.io.FileUtil; -import org.cobbzilla.util.io.Tarball; -import org.cobbzilla.util.io.TempDir; -import org.cobbzilla.util.string.Base64; -import org.hibernate.criterion.Order; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Repository; - -import java.io.*; -import java.util.List; - -import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX; -import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE; -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; -import static org.cobbzilla.util.string.Base64.DONT_GUNZIP; -import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; - -@Repository @Slf4j -public class AnsibleRoleDAO extends AccountOwnedTemplateDAO { - - public static final String ROLE_PATH = "automation/roles/"; - - @Autowired private CloudServiceDAO cloudDAO; - @Autowired private StorageService storageService; - @Autowired private BubbleConfiguration configuration; - - @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; } - - @Override public Object preCreate(AnsibleRole role) { - - // convert [[ .. ]] to {{ }} - if (role.hasConfig()) role.setConfigJson(role.getConfigJson().replace("[[", "{{").replace("]]", "}}")); - - if (!role.hasTgzB64()) throw invalidEx("err.tgzB64.required"); - - // Is it a raw tgz? - if (role.isTgzB64raw()) { - // Ensure it is a valid tarball - try { - @Cleanup("delete") final TempDir temp = new TempDir(); - final File tarballFile = new File(temp, role.getName() + ".tgz"); - FileUtil.toFile(tarballFile, new ByteArrayInputStream(Base64.decode(role.getTgzB64(), DONT_GUNZIP))); - - Tarball.unroll(tarballFile, temp); - final File namedRoleDir = new File(temp, role.getRoleName()); - if (!namedRoleDir.exists() || !namedRoleDir.isDirectory()) throw invalidEx("err.tgzB64.invalid.noRolesDir", "no roles/"+ role.getName()+" dir"); - - // the role dir and the tarball are the only 2 files here - final String[] files = temp.list(); - if (files == null || files.length != 2) { - throw invalidEx("err.tgzB64.invalid.wrongNumberOfFiles", "multiple entries in tarball base directory"); - } - - final File mainTask = new File(new File(namedRoleDir, "tasks"), "main.yml"); - if (!mainTask.exists() || !mainTask.isFile()) throw invalidEx("err.tgzB64.invalid.missingTasksMainYml", "no roles/"+ role.getName()+"/tasks/main.yml file"); - - final String key = ROLE_PATH + tarballFile.getName(); - final List clouds = cloudDAO.findByAccountAndType(role.getAccount(), CloudServiceType.storage); - String stored = null; - for (CloudService cloud : clouds) { - try { - @Cleanup final FileInputStream in = new FileInputStream(tarballFile); - cloud.getStorageDriver(configuration).write(configuration.getThisNode().getUuid(), key, in); - stored = STORAGE_PREFIX+cloud.getName()+"/"+key; - break; - } catch (Exception e) { - log.warn("preCreate: error storing role archive to "+cloud.getName()+"/"+key+": "+e); - } - } - if (stored == null) { - return die("preCreate: failed to store role archive to any storage service"); - } - role.setTgzB64(stored); - - } catch (Exception e) { - throw invalidEx("err.tgzB64.invalid.writingToStorage", "error validating tarball/writing to storage: "+e); - } - - } else if (role.isTgzB64storage()) { - // Verify file exists in storage - try { - if (!storageService.exists(role.getAccount(), role.getTgzB64())) { - throw new IllegalStateException("preCreate: role archive not found in storage: "+role.getTgzB64()); - } - } catch (Exception e) { - boolean existsOnClasspath = false; - final String prefix = STORAGE_PREFIX + LOCAL_STORAGE + "/"; - final String roleTgzPath; - if (role.getTgzB64().startsWith(prefix + "automation/roles/")) { - // check classpath - roleTgzPath = role.getTgzB64().substring(prefix.length()); - try { - @Cleanup final InputStream in = getClass().getClassLoader().getResourceAsStream(roleTgzPath); - existsOnClasspath = in != null; - if (existsOnClasspath) { - if (!storageService.write(role.getAccount(), role.getTgzB64(), in)) { - log.warn("preCreate: error writing role archive from classpath:"+roleTgzPath+" -> storage:"+role.getTgzB64()); - } - } - } catch (Exception ioe) { - log.warn("preCreate: role archive not found in storage ("+role.getTgzB64()+") and exception searching classpath ("+roleTgzPath+"): "+shortError(ioe)); - } - } else { - roleTgzPath = null; - } - if (!existsOnClasspath) { - throw invalidEx("err.tgzB64.invalid.readingFromStorage", "error reading from " + roleTgzPath + " : " + e); - } - } - } - - return super.preCreate(role); - } - - public List findByAccountAndNames(Account account, String[] roles) { - return findByFieldAndFieldIn("account", account.getUuid(), "name", roles, getDefaultSortOrder()); - } - -} diff --git a/bubble-server/src/main/java/bubble/dao/cloud/BubbleDomainDAO.java b/bubble-server/src/main/java/bubble/dao/cloud/BubbleDomainDAO.java index 5fab1229..3598f467 100644 --- a/bubble-server/src/main/java/bubble/dao/cloud/BubbleDomainDAO.java +++ b/bubble-server/src/main/java/bubble/dao/cloud/BubbleDomainDAO.java @@ -4,9 +4,7 @@ */ package bubble.dao.cloud; -import bubble.dao.account.AccountDAO; import bubble.dao.account.AccountOwnedTemplateDAO; -import bubble.model.account.Account; import bubble.model.cloud.BubbleDomain; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; @@ -15,7 +13,6 @@ import org.hibernate.criterion.Order; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; -import java.util.ArrayList; import java.util.List; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @@ -23,37 +20,16 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Repository @Slf4j public class BubbleDomainDAO extends AccountOwnedTemplateDAO { - @Autowired private AnsibleRoleDAO roleDAO; @Autowired private CloudServiceDAO cloudDAO; - @Autowired private AccountDAO accountDAO; @Autowired private BubbleConfiguration configuration; @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; } @Override public Object preCreate(BubbleDomain domain) { - final String[] roles = domain.getRoles(); - if (roles == null || roles.length == 0) throw invalidEx("err.roles.required"); - final CloudService dnsService = cloudDAO.findByUuid(domain.getPublicDns()); if (dnsService == null) throw invalidEx("err.dns.notFound", "cloud service not found: "+domain.getPublicDns(), domain.getPublicDns()); - final Account account = accountDAO.findByUuid(domain.getAccount()); - final List validRoles = new ArrayList<>(); - for (String r : roles) { - if (roleDAO.findByAccountAndId(domain.getAccount(), r) == null) { - if (!account.hasParent()) { - validRoles.add(r); - } else if (roleDAO.findPublicTemplate(account.getParent(), r) == null) { - continue; - } - validRoles.add(r); - } else { - validRoles.add(r); - } - } - - domain.setRoles(validRoles.toArray(new String[0])); return super.preCreate(domain); } diff --git a/bubble-server/src/main/java/bubble/model/account/Account.java b/bubble-server/src/main/java/bubble/model/account/Account.java index d4fcf57e..d1f77795 100644 --- a/bubble-server/src/main/java/bubble/model/account/Account.java +++ b/bubble-server/src/main/java/bubble/model/account/Account.java @@ -66,7 +66,6 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @ECTypeChild(type=RuleDriver.class, backref="account"), @ECTypeChild(type=BubbleApp.class, backref="account"), @ECTypeChild(type=AppData.class, backref="account"), - @ECTypeChild(type=AnsibleRole.class, backref="account"), @ECTypeChild(type=CloudService.class, backref="account"), @ECTypeChild(type=BubbleFootprint.class, backref="account"), @ECTypeChild(type=BubbleDomain.class, backref="account"), diff --git a/bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java b/bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java index a9eb0f1a..cc92bb4d 100644 --- a/bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java +++ b/bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java @@ -5,7 +5,6 @@ package bubble.model.boot; import bubble.model.account.AccountSshKey; -import bubble.model.cloud.AnsibleRole; import bubble.model.cloud.BubbleDomain; import bubble.model.cloud.CloudService; import lombok.Getter; @@ -37,9 +36,6 @@ public class ActivationRequest { @HasValue(message="err.networkName.required") @Getter @Setter private String networkName = "boot-network"; - @Getter @Setter private AnsibleRole[] roles; - public boolean hasRoles () { return !empty(roles); } - @Getter @Setter private Map cloudConfigs = new LinkedHashMap<>(); public boolean hasCloudConfigs () { return !empty(cloudConfigs); } public ActivationRequest addCloudConfig(CloudService cloud) { diff --git a/bubble-server/src/main/java/bubble/model/cloud/AnsibleRole.java b/bubble-server/src/main/java/bubble/model/cloud/AnsibleRole.java deleted file mode 100644 index 1b7932f6..00000000 --- a/bubble-server/src/main/java/bubble/model/cloud/AnsibleRole.java +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.model.cloud; - -import bubble.model.account.Account; -import bubble.model.account.AccountTemplate; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; -import org.cobbzilla.util.collection.HasPriority; -import org.cobbzilla.util.collection.NameAndValue; -import org.cobbzilla.wizard.model.IdentifiableBase; -import org.cobbzilla.wizard.model.SemanticVersion; -import org.cobbzilla.wizard.model.entityconfig.annotations.*; -import org.cobbzilla.wizard.validation.HasValue; - -import javax.persistence.*; -import javax.validation.constraints.Pattern; -import javax.validation.constraints.Size; -import java.util.Arrays; - -import static bubble.ApiConstants.DB_JSON_MAPPER; -import static bubble.ApiConstants.EP_ROLES; -import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.reflect.ReflectionUtil.copy; - -@ECType(root=true, name="role") -@ECTypeURIs(baseURI=EP_ROLES, listFields={"account", "name", "description"}) -@Entity @NoArgsConstructor @Accessors(chain=true) -@ECIndexes({ - @ECIndex(unique=true, of={"account", "name"}), - @ECIndex(of={"account", "template", "enabled"}), - @ECIndex(of={"template", "enabled"}) -}) -public class AnsibleRole extends IdentifiableBase implements AccountTemplate, HasPriority { - - public static final String[] COPY_FIELDS = { - "name", "priority", "description", "template", "enabled", - "configJson", "optionalConfigNamesJson", "tgzB64" - }; - public static final String ROLENAME_PATTERN = "[_A-Za-z0-9]+-[\\d]+\\.[\\d]+\\.[\\d]+"; - - public AnsibleRole(AnsibleRole role) { copy(this, role, COPY_FIELDS); } - - @ECSearchable(filter=true) @ECField(index=10) - @HasValue(message="err.name.required") - @Pattern(regexp=ROLENAME_PATTERN, message="err.name.invalid") - @ECIndex @Column(nullable=false, updatable=false, length=200) - @Getter @Setter private String name; - - @ECSearchable @ECField(index=20) - @ECForeignKey(entity=Account.class) - @Column(nullable=false, updatable=false, length=UUID_MAXLEN) - @Getter @Setter private String account; - - @JsonIgnore @Transient @Getter(lazy=true) private final String roleName = getRoleName(name); - @JsonIgnore @Transient @Getter(lazy=true) private final SemanticVersion version = getRoleVersion(name); - - public static String getRoleName(String name) { - final int lastHyphen = name.lastIndexOf('-'); - return lastHyphen == -1 ? name : name.substring(0, lastHyphen); - } - public static boolean sameRoleName(String r1, String r2) { return getRoleName(r1).equals(getRoleName(r2)); } - - public static SemanticVersion getRoleVersion(String name) { - final int lastHyphen = name.lastIndexOf('-'); - return lastHyphen == -1 ? null : new SemanticVersion(name.substring(lastHyphen +1)); - } - - public static String findRole(String[] roles, String role) { - if (roles == null) return null; - return Arrays.stream(roles) - .filter(r -> sameRoleName(role, r)) - .findAny() - .orElse(null); - } - - @ECSearchable(filter=true) @ECField(index=30) - @Size(max=10000, message="err.description.length") - @Getter @Setter private String description; - - @ECSearchable @ECField(index=40) - @Enumerated(EnumType.STRING) - @ECIndex @Column(nullable=false, length=20) - @Getter @Setter private AnsibleInstallType install = AnsibleInstallType.standard; - public boolean shouldInstall(AnsibleInstallType installType) { - return install.shouldInstall(installType); - } - - @ECSearchable @ECField(index=50) - @ECIndex @Column(nullable=false) - @Getter @Setter private Integer priority; - - @ECSearchable @ECField(index=60) - @ECIndex @Column(nullable=false) - @Getter @Setter private Boolean template = false; - - @ECSearchable @ECField(index=70) - @ECIndex @Column(nullable=false) - @Getter @Setter private Boolean enabled = true; - - @Column(updatable=false, length=10000) - @JsonIgnore @Getter @Setter private String configJson; - public boolean hasConfig () { return configJson != null; } - - @Transient public NameAndValue[] getConfig () { return configJson == null ? null : json(configJson, NameAndValue[].class); } - public AnsibleRole setConfig(NameAndValue[] config) { return setConfigJson(config == null ? null : json(config, DB_JSON_MAPPER)); } - - @Column(updatable=false, length=1000) @ECField(index=80) - @JsonIgnore @Getter @Setter private String optionalConfigNamesJson; - public boolean hasOptionalConfigNames () { return optionalConfigNamesJson != null; } - - @Transient public String[] getOptionalConfigNames() { return optionalConfigNamesJson == null ? null : json(optionalConfigNamesJson, String[].class); } - public AnsibleRole setOptionalConfigNames(String[] names) { return setOptionalConfigNamesJson(name == null ? null : json(names, DB_JSON_MAPPER)); } - - // The Base64-encoded .tgz archive for the role directory. all paths should start with roles//... - // Then after it is stored (in AnsibleRoleDAO.preCreate), this becomes storage://CloudServiceName/path - @Column(updatable=false, length=200) - @Getter @Setter private String tgzB64; - public boolean hasTgzB64 () { return !empty(tgzB64); } - - @Transient @JsonIgnore public boolean isTgzB64raw() { return tgzB64 != null && !tgzB64.startsWith(STORAGE_PREFIX); } - @Transient @JsonIgnore public boolean isTgzB64storage() { return tgzB64 != null && tgzB64.startsWith(STORAGE_PREFIX); } - - public boolean isOptionalConfigName(String cfgName) { - final String[] names = getOptionalConfigNames(); - if (names == null) return false; - return Arrays.asList(names).contains(cfgName); - } -} diff --git a/bubble-server/src/main/java/bubble/model/cloud/BubbleDomain.java b/bubble-server/src/main/java/bubble/model/cloud/BubbleDomain.java index d1f10165..ff6d9b25 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/BubbleDomain.java +++ b/bubble-server/src/main/java/bubble/model/cloud/BubbleDomain.java @@ -6,7 +6,6 @@ package bubble.model.cloud; import bubble.model.account.Account; import bubble.model.account.AccountTemplate; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -24,20 +23,12 @@ import org.hibernate.annotations.Type; import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.Transient; import javax.validation.constraints.Size; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import static bubble.ApiConstants.DB_JSON_MAPPER; import static bubble.ApiConstants.EP_DOMAINS; -import static bubble.model.cloud.AnsibleRole.sameRoleName; import static org.apache.commons.lang3.StringUtils.countMatches; -import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.dns.DnsType.NS; import static org.cobbzilla.util.dns.DnsType.SOA; -import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @@ -57,8 +48,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; public class BubbleDomain extends IdentifiableBase implements AccountTemplate, HasPriority { public static final String[] UPDATE_FIELDS = {"description", "template", "enabled", "priority", "publicDns"}; - public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, - "name", "rolesJson"); + public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "name"); public static final int DOMAIN_NAME_MAXLEN = 200; @@ -108,49 +98,6 @@ public class BubbleDomain extends IdentifiableBase implements AccountTemplate, H @Column(nullable=false, length=UUID_MAXLEN) @Getter @Setter private String publicDns; - @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL") - @JsonIgnore @Getter @Setter private String rolesJson; - public boolean hasRoles () { return !empty(getRoles()); } - - @Transient public String[] getRoles () { return rolesJson == null ? null : json(rolesJson, String[].class); } - public BubbleDomain setRoles (String[] roles) { return setRolesJson(roles == null ? null : json(roles, DB_JSON_MAPPER)); } - - public String findRole(String r) { return AnsibleRole.findRole(getRoles(), r); } - - public BubbleDomain addRole(String r) { - final String[] roles = getRoles(); - if (roles != null) { - if (Arrays.stream(roles).noneMatch(role -> sameRoleName(role, r))) { - return setRoles(ArrayUtil.append(roles, r)); - } else { - log.warn("addRole("+r+"): role already exists in domain "+getName()); - return this; - } - } else { - return setRoles(new String[]{r}); - } - } - - public BubbleDomain removeRole(String r) { - final String[] roles = getRoles(); - if (roles == null) return null; - - final List newRoles = Arrays.stream(roles).filter(role -> !sameRoleName(role, r)).collect(Collectors.toList()); - return setRolesJson(json(newRoles)); - } - - public BubbleDomain updateRole(String previous, String current) { - final String[] roles = getRoles(); - if (roles == null) return die("updateRole: no roles!"); - final String role = findRole(previous); - if (role != null) { - log.debug("updateRole: removing previous role: "+role); - removeRole(role); - } - log.debug("updateRole: adding new role: "+current); - return addRole(current); - } - public String networkFromFqdn(String fqdn, ValidationResult errors) { if (!fqdn.endsWith("."+getName())) { errors.addViolation("err.fqdn.outOfNetwork"); diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java index f3a352dd..06ad3aef 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java @@ -518,13 +518,6 @@ public class AccountsResource { return ok(BubbleDeviceType.getSelectableTypes()); } - @Path("/{id}"+EP_ROLES) - public AnsibleRolesResource getAnsibleRoles(@Context ContainerRequest ctx, - @PathParam("id") String id) { - final AccountContext c = new AccountContext(ctx, id); - return configuration.subResource(AnsibleRolesResource.class, c.account); - } - @Path("/{id}"+EP_VPN) public VpnConfigResource getVpnConfig(@Context ContainerRequest ctx, @PathParam("id") String id) { diff --git a/bubble-server/src/main/java/bubble/resources/account/MeResource.java b/bubble-server/src/main/java/bubble/resources/account/MeResource.java index 907aaeb6..585daa0d 100644 --- a/bubble-server/src/main/java/bubble/resources/account/MeResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/MeResource.java @@ -278,12 +278,6 @@ public class MeResource { return configuration.subResource(NetworksResource.class, caller); } - @Path(EP_ROLES) - public AnsibleRolesResource getAnsibleRoles(@Context ContainerRequest ctx) { - final Account caller = userPrincipal(ctx); - return configuration.subResource(AnsibleRolesResource.class, caller); - } - @Path(EP_SENT_NOTIFICATIONS) public SentNotificationsResource getSentNotificationsResource(@Context ContainerRequest ctx) { final Account caller = userPrincipal(ctx); diff --git a/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResource.java b/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResource.java deleted file mode 100644 index 5670862f..00000000 --- a/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResource.java +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.resources.cloud; - -import bubble.model.account.Account; - -public class AnsibleRolesResource extends AnsibleRolesResourceBase { - - public AnsibleRolesResource(Account account) { super(account); } - -} diff --git a/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResourceBase.java b/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResourceBase.java deleted file mode 100644 index a7f55ac5..00000000 --- a/bubble-server/src/main/java/bubble/resources/cloud/AnsibleRolesResourceBase.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.resources.cloud; - -import bubble.dao.cloud.AnsibleRoleDAO; -import bubble.model.account.Account; -import bubble.model.cloud.AnsibleRole; -import bubble.resources.account.AccountOwnedTemplateResource; -import org.glassfish.grizzly.http.server.Request; -import org.glassfish.jersey.server.ContainerRequest; - -import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; - -public class AnsibleRolesResourceBase extends AccountOwnedTemplateResource { - - public AnsibleRolesResourceBase(Account account) { super(account); } - - @Override protected boolean canCreate(Request req, ContainerRequest ctx, Account caller, AnsibleRole request) { - // ensure a role with the same name/version does not exist for this account - final AnsibleRole existing = getDao().findByAccountAndId(caller.getUuid(), request.getName()); - if (existing != null) throw invalidEx("err.role.exists", "A role exists with name: "+request.getName()); - return super.canCreate(req, ctx, caller, request); - } - - @Override protected boolean canUpdate(ContainerRequest ctx, Account caller, AnsibleRole found, AnsibleRole request) { - // roles cannot be updated - return false; - } - -} diff --git a/bubble-server/src/main/java/bubble/resources/cloud/DomainRolesResource.java b/bubble-server/src/main/java/bubble/resources/cloud/DomainRolesResource.java deleted file mode 100644 index e999b907..00000000 --- a/bubble-server/src/main/java/bubble/resources/cloud/DomainRolesResource.java +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.resources.cloud; - -import bubble.dao.cloud.AnsibleRoleDAO; -import bubble.dao.cloud.BubbleDomainDAO; -import bubble.model.account.Account; -import bubble.model.cloud.AnsibleRole; -import bubble.model.cloud.BubbleDomain; -import org.cobbzilla.wizard.model.SemanticVersion; -import org.glassfish.jersey.server.ContainerRequest; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.Response; - -import static bubble.model.cloud.AnsibleRole.*; -import static bubble.resources.account.AccountOwnedResource.validateAccountUuid; -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; -import static org.cobbzilla.wizard.resources.ResourceUtil.*; - -@Consumes(APPLICATION_JSON) -@Produces(APPLICATION_JSON) -public class DomainRolesResource { - - public Account account; - public BubbleDomain domain; - - @Autowired private BubbleDomainDAO domainDAO; - @Autowired private AnsibleRoleDAO roleDAO; - - public DomainRolesResource(Account account, BubbleDomain domain) { - this.account = account; - this.domain = domain; - } - - @GET - public Response list(@Context ContainerRequest ctx) { - final DomainRoleContext drc = new DomainRoleContext(ctx); - return ok(drc.getDomain().getRoles()); - } - - @GET @Path("/{role}") - public Response find(@Context ContainerRequest ctx, - @PathParam("role") String role) { - final DomainRoleContext drc = new DomainRoleContext(ctx, role); - if (drc.domainRole == null) return notFound(role); - return ok(drc.domainRole); - } - - @PUT @Path("/{role}") - public Response add(@Context ContainerRequest ctx, - @PathParam("role") String role) { - - final DomainRoleContext drc = new DomainRoleContext(ctx, role); - - // does the domain already have a role with the same name? - if (drc.domainRole != null) { - final SemanticVersion version = getRoleVersion(role); - final SemanticVersion existingVersion = getRoleVersion(drc.domainRole); - if (existingVersion == null) return die("add: role defined without version in domain: "+drc.getDomain().getUuid()); - if (existingVersion.equals(version)) { - // same version, nothing to do - return ok(drc.domainRoles); - - } else { - // different version, cannot add it, it's already there - return invalid("err.domainRole.alreadyExists", "Cannot add role "+role+" to domain "+domain.getName()+", already includes role: "+drc.domainRole); - } - } - - // role is not in domain's role list. add it, commit the domain - final BubbleDomain updated = domainDAO.update(drc.getDomain().addRole(drc.role.getName())); - return ok(updated.getRoles()); - } - - @POST @Path("/{role}") - public Response update(@Context ContainerRequest ctx, - @PathParam("role") String roleName, - AnsibleRole role) { - // if roleName path param has a version, use it. otherwise use the version found in the role entity - SemanticVersion version = getRoleVersion(roleName); - if (version != null) { - // ensure it matches what is in the entity - if (!role.getVersion().equals(version)) return versionMismatch(roleName, role); - } else { - version = role.getVersion(); - roleName = getRoleName(roleName)+"-"+version; - } - - // roleName must match json - if (!sameRoleName(roleName, role.getName())) return invalid("err.role.invalid", "role name mismatch", roleName); - - final DomainRoleContext drc = new DomainRoleContext(ctx, roleName, true); - if (drc.domainRole == null) { - log.warn("update: role not found, adding: "+roleName); - } else { - // if version is the same, no changes to make - if (version.equals(getRoleVersion(drc.domainRole))) { - log.info("update: same version, not updating"); - return ok(drc.domainRoles); - } - } - - final BubbleDomain updated = domainDAO.update(drc.getDomain().updateRole(roleName, role.getName())); - log.debug("update: updated.roles="+updated.getRolesJson()); - return ok(updated.getRoles()); - } - - public Response versionMismatch(@PathParam("role") String roleName, AnsibleRole role) { - return invalid("err.version.mismatch", "version in URL ("+roleName+") did not match version in object ("+role.getVersion()+")", roleName); - } - - @DELETE @Path("/{role}") - public Response remove(@Context ContainerRequest ctx, - @PathParam("role") String role) { - - final DomainRoleContext drc = new DomainRoleContext(ctx, role); - if (drc.domainRole == null) return notFound(role); - - final BubbleDomain updated = domainDAO.update(drc.getDomain().removeRole(role)); - return ok(updated.getRoles()); - } - - private class DomainRoleContext { - - public Account caller; - private BubbleDomain d; - public BubbleDomain getDomain () { return d; } - public AnsibleRole role; - public String[] domainRoles; - public String domainRole; - - public DomainRoleContext(ContainerRequest ctx) { this(ctx, null); } - - public DomainRoleContext(ContainerRequest ctx, String roleName) { - this(ctx, roleName, false); - } - public DomainRoleContext(ContainerRequest ctx, String roleName, boolean okNotFound) { - - caller = userPrincipal(ctx); - final String accountUuid = validateAccountUuid(account, ctx, caller); - - d = domainDAO.findByUuid(domain.getUuid()); - if (d == null) throw notFoundEx(domain.getName()); - - if (roleName != null) { - role = roleDAO.findByAccountAndId(accountUuid, roleName); - if (role == null) throw notFoundEx(roleName); - - domainRoles = d.getRoles(); - domainRole = findRole(domainRoles, roleName); - - // user was requesting exact version -- ensure the domainRole version matches - if (!okNotFound && roleName.contains("-") && !roleName.equals(domainRole)) throw notFoundEx(roleName); - } - } - } -} diff --git a/bubble-server/src/main/java/bubble/resources/cloud/DomainsResourceBase.java b/bubble-server/src/main/java/bubble/resources/cloud/DomainsResourceBase.java index 824d59c7..a64207ef 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/DomainsResourceBase.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/DomainsResourceBase.java @@ -54,13 +54,4 @@ public class DomainsResourceBase extends AccountOwnedTemplateResource roles, ValidationResult errors, - File tarballDir, boolean fork, - String restoreKey) throws IOException, ArchiveException { + String restoreKey) throws IOException { final BubbleConfiguration c = configuration; - if (tarballDir == null) tarballDir = automation; final AnsibleInstallType installType = network.getInstallType(); - roles.sort(SORT_PRIORITY); - final List installRoles = roles.stream() - .filter(role -> role.shouldInstall(installType)) - .collect(Collectors.toList()); + final String[] installRoles = installType == AnsibleInstallType.sage + ? ApiConstants.ROLES_SAGE + : ApiConstants.ROLES_NODE; final Map ctx = new HashMap<>(); final Handlebars handlebars = c.getHandlebars(); @@ -94,7 +90,7 @@ public class AnsiblePrepService { ctx.put("network", network); ctx.put("node", node); - ctx.put("roles", installRoles.stream().map(AnsibleRole::getRoleName).collect(Collectors.toList())); + ctx.put("roles", installRoles); ctx.put("testMode", !fork && configuration.testMode()); // Determine which apps should be copied based on plan @@ -108,34 +104,27 @@ public class AnsiblePrepService { } // Copy database with new encryption key - if (installRoles.stream().anyMatch(r->r.getName().startsWith("bubble-"))) { - final String key = dbFilter.copyDatabase(fork, network, node, account, planApps, new File(bubbleFilesDir, "bubble.sql.gz")); - ctx.put("dbEncryptionKey", key); - - // if this is a fork, and current server is local, then sage will be self - if (fork && configuration.getThisNode().localIp4()) { - ctx.put("sageNode", node.getUuid()); - } else { - // otherwise, sage will be us, the node that is launching the new node - ctx.put("sageNode", configuration.getThisNode().getUuid()); - } - } + final String key = dbFilter.copyDatabase(fork, network, node, account, planApps, new File(bubbleFilesDir, "bubble.sql.gz")); + ctx.put("dbEncryptionKey", key); - for (AnsibleRole role : roles) { - @Cleanup final InputStream roleStream = getTgzInputStream(node.getAccount(), role); - if (roleStream == null) { - errors.addViolation("err.role.notFound", "roleStream was null for: "+role.getTgzB64()); - continue; - } - final File roleTarball = toFile(new File(tarballDir, role.getName() + ".tgz"), roleStream); - final File rolesDir = new File(automation, "roles"); - Tarball.unroll(roleTarball, rolesDir); + // if this is a fork, and current server is local, then sage will be self + if (fork && configuration.getThisNode().localIp4()) { + ctx.put("sageNode", node.getUuid()); + } else { + // otherwise, sage will be us, the node that is launching the new node + ctx.put("sageNode", configuration.getThisNode().getUuid()); + } - if (role.hasConfig()) { - final String roleName = role.getRoleName(); - final File roleDir = new File(abs(rolesDir)+"/"+roleName); + final File rolesDir = new File(automation, "roles"); + for (String roleName : installRoles) { + final TempDir roleTemp = copyClasspathDirectory("ansible/roles/"+roleName); + final File roleDir = new File(rolesDir, roleName); + if (!roleTemp.renameTo(roleDir)) return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir)); + final File bubbleRoleJson = new File(abs(roleDir)+"/files/bubble_role.json"); + if (bubbleRoleJson.exists()) { final File varsDir = mkdirOrDie(new File(abs(roleDir)+"/vars")); final File varsMain = new File(varsDir, "main.yml"); + final AnsibleRole role = json(FileUtil.toStringOrDie(bubbleRoleJson), AnsibleRole.class); try (Writer w = new FileWriter(varsMain)) { for (NameAndValue cfg : role.getConfig()) { final String cfgName = cfg.getName(); @@ -158,19 +147,8 @@ public class AnsiblePrepService { return ctx; } - public InputStream getTgzInputStream(String account, AnsibleRole role) { - try { - final String tgzB64 = role.getTgzB64(); - if (role.isTgzB64storage()) { - return storageService.read(role.getAccount(), role.getTgzB64()); - - } else { - log.debug("getTgzInputStream: reading directly from tgzB64"); - return new ByteArrayInputStream(Base64.decode(tgzB64, DONT_GUNZIP)); - } - } catch (Exception e) { - return die("getTgzInputStream: "+e, e); - } + public static void main (String[] args) { + final TempDir tempDir = copyClasspathDirectory("ansible/roles/common"); + log.info("tempDir="+abs(tempDir)); } - } diff --git a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterConstants.java b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterConstants.java index 6961887b..9716de9e 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterConstants.java +++ b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterConstants.java @@ -47,7 +47,6 @@ public class NodeProgressMeterConstants { public static final String METER_ERROR_PEER_LIMIT_REACHED = "BUBBLE-ERROR: PEER LIMIT REACHED"; public static final String METER_ERROR_NODE_CLOUD_NOT_FOUND = "BUBBLE-ERROR: NODE CLOUD NOT FOUND"; public static final String METER_ERROR_BUBBLE_JAR_NOT_FOUND = "BUBBLE-ERROR: BUBBLE JAR NOT FOUND"; - public static final String METER_ERROR_ROLES_NOT_FOUND = "BUBBLE-ERROR: ANSIBLE ROLES NOT FOUND"; public static final String METER_ERROR_NO_IP_OR_SSH_KEY = "BUBBLE-ERROR: NODE STARTED BUT HAS NO IP ADDRESS OR SSH KEY"; public static final String METER_ERROR_ROLE_VALIDATION_ERRORS = "BUBBLE-ERROR: ROLE VALIDATION FAILED"; diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index 9b116e8a..f420ac54 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -113,7 +113,6 @@ public class StandardNetworkService implements NetworkService { @Autowired private BubbleDomainDAO domainDAO; @Autowired private CloudServiceDAO cloudDAO; @Autowired private BubbleConfiguration configuration; - @Autowired private AnsibleRoleDAO roleDAO; @Autowired private AccountPlanDAO accountPlanDAO; @Autowired private AccountPolicyDAO policyDAO; @Autowired private AccountMessageDAO accountMessageDAO; @@ -224,15 +223,8 @@ public class StandardNetworkService implements NetworkService { @Cleanup("delete") final TempDir automation = new TempDir(); final File bubbleFilesDir = mkdirOrDie(new File(abs(automation) + "/roles/bubble/files")); - final List roles = roleDAO.findByAccountAndNames(account, domain.getRoles()); - if (roles.size() != domain.getRoles().length) { - progressMeter.error(METER_ERROR_ROLES_NOT_FOUND); - return die("newNode: error finding ansible roles"); - } - // build automation directory for this run final ValidationResult errors = new ValidationResult(); - final File roleTgzDir = mkdirOrDie(new File(abs(bubbleFilesDir), "role_tgz")); progressMeter.write(METER_TICK_LAUNCHING_NODE); node.setState(BubbleNodeState.starting); @@ -258,7 +250,7 @@ public class StandardNetworkService implements NetworkService { progressMeter.write(METER_TICK_PREPARING_ROLES); final Map ctx = ansiblePrep.prepAnsible( automation, bubbleFilesDir, account, network, node, computeDriver, - roles, errors, roleTgzDir, nn.fork(), nn.getRestoreKey()); + errors, nn.fork(), nn.getRestoreKey()); if (errors.isInvalid()) { progressMeter.error(METER_ERROR_ROLE_VALIDATION_ERRORS); throw new MultiViolationException(errors.getViolationBeans()); diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json new file mode 100644 index 00000000..5b603a86 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json @@ -0,0 +1,33 @@ +{ + "name": "bubble", + "config": [ + {"name": "node_uuid", "value": "[[node.uuid]]"}, + {"name": "network_uuid", "value": "[[node.network]]"}, + {"name": "admin_port", "value": "[[node.adminPort]]"}, + {"name": "ssl_port", "value": "[[node.sslPort]]"}, + {"name": "public_base_uri", "value": "[[publicBaseUri]]"}, + {"name": "sage_node", "value": "[[sageNode]]"}, + {"name": "install_type", "value": "[[installType]]"}, + {"name": "promo_code_policy", "value": "[[#compare fork '==' true]][[configuration.promoCodePolicy]][[else]]disabled[[/compare]]"}, + {"name": "default_locale", "value": "[[network.locale]]"}, + {"name": "time_zone", "value": "[[network.timezone]]"}, + {"name": "bubble_version", "value": "[[configuration.version]]"}, + {"name": "bubble_host", "value": "[[node.fqdn]]"}, + {"name": "bubble_cname", "value": "[[network.networkDomain]]"}, + {"name": "admin_user", "value": "[[node.ansibleUser]]"}, + {"name": "db_encoding", "value": "UTF-8"}, + {"name": "db_locale", "value": "en_US"}, + {"name": "db_user", "value": "bubble"}, + {"name": "db_name", "value": "bubble"}, + {"name": "db_key", "value": "[[dbEncryptionKey]]"}, + {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, + {"name": "is_fork", "value": "[[fork]]"}, + {"name": "restore_key", "value": "[[restoreKey]]"}, + {"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"}, + {"name": "test_mode", "value": "[[testMode]]"}, + {"name": "error_url", "value": "[[error_url]]"}, + {"name": "error_key", "value": "[[error_key]]"}, + {"name": "error_env", "value": "[[error_env]]"} + ], + "optionalConfigNames": ["restore_key", "restore_timeout", "error_url", "error_key", "error_env"] +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml new file mode 100644 index 00000000..84c1c38a --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml @@ -0,0 +1,64 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Set system timezone + shell: timedatectl set-timezone {{ time_zone }} + +- name: Install helper template scripts + template: + src: "{{ item.src }}" + dest: "/usr/local/bin/{{ item.dest }}" + owner: root + group: root + mode: 0555 + with_items: + - { src: "full_reset_db.sh.j2", dest: "full_reset_db.sh" } + - { src: "snapshot_ansible.sh.j2", dest: "snapshot_ansible.sh" } + +- name: Generate keys + shell: random_password.sh /home/bubble/{{ item.file }} bubble {{ item.group }} + with_items: + - { file: '.BUBBLE_REDIS_ENCRYPTION_KEY', group: root } + - { file: '.BUBBLE_DB_ENCRYPTION_KEY', group: postgres } # postgres user needs access to DB key + - { file: '.BUBBLE_PG_PASSWORD', group: postgres } # postgres user needs access to DB password + +- name: Write DB key + shell: echo -n "{{ db_key }}" > /home/bubble/.BUBBLE_DB_ENCRYPTION_KEY + +- name: Write bubble env file + template: + src: bubble.env.j2 + dest: /home/bubble/bubble_{{ bubble_version }}/bubble.env + owner: bubble + group: bubble + mode: 0400 + +- name: Install bubble self_node.json, sage_node.json and sage_key.json + copy: + src: "{{ item }}" + dest: /home/bubble/{{ item }} + owner: bubble + group: bubble + mode: 0600 + with_items: + - self_node.json + - sage_node.json + - sage_key.json + +- name: Initialize local storage with role archive + shell: init_roles.sh + +- import_tasks: postgresql_data.yml + +- name: Install refresh_bubble_ssh_keys script + template: + src: refresh_bubble_ssh_keys.sh.j2 + dest: /usr/local/sbin/refresh_bubble_ssh_keys.sh + owner: root + group: root + mode: 0500 + +- name: Install refresh_bubble_ssh_keys_monitor supervisor conf file + copy: + src: supervisor_refresh_bubble_ssh_keys_monitor.conf + dest: /etc/supervisor/conf.d/refresh_bubble_ssh_keys_monitor.conf diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/postgresql_data.yml b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/postgresql_data.yml new file mode 100644 index 00000000..164b488a --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/postgresql_data.yml @@ -0,0 +1,15 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install SQL schema files + copy: + src: "{{ item }}" + dest: /home/bubble/sql/{{ item }} + owner: bubble + group: postgres + mode: 0440 + with_items: + - "bubble.sql.gz" + +- name: Populate database + shell: sudo -H -u postgres bash -c "cd && full_reset_db.sh" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml new file mode 100644 index 00000000..61f57fd1 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml @@ -0,0 +1,17 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install restore helper scripts + copy: + src: '{{ item }}' + dest: "/usr/local/bin/{{ item }}" + owner: root + group: postgres + mode: 0550 + with_items: + - "bubble_restore_monitor.sh" + when: restore_key is defined + +- name: Start restore monitor + shell: bash -c 'nohup /usr/local/bin/bubble_restore_monitor.sh {{ admin_port }} {{ restore_timeout }} > /dev/null &' + when: restore_key is defined diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 b/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 new file mode 100644 index 00000000..19c47f68 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 @@ -0,0 +1,12 @@ +export PUBLIC_BASE_URI={{ public_base_uri }} +export BUBBLE_ASSETS_DIR=/home/bubble/site +export SELF_NODE={{ node_uuid }} +export SAGE_NODE={{ sage_node }} +export BUBBLE_PROMO_CODE_POLICY={{ promo_code_policy }} +export LETSENCRYPT_EMAIL={{ letsencrypt_email }} +export BUBBLE_SERVER_PORT={{ admin_port }} +export BUBBLE_TEST_MODE={{ test_mode }} +export BUBBLE_DEFAULT_LOCALE={{ default_locale }} +export ERRBIT_URL={{ error_url | default('') }} +export ERRBIT_KEY={{ error_key | default('') }} +export ERRBIT_ENV={{ error_env | default('') }} diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/templates/full_reset_db.sh.j2 b/bubble-server/src/main/resources/ansible/roles/bubble/templates/full_reset_db.sh.j2 new file mode 100644 index 00000000..896cad07 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/templates/full_reset_db.sh.j2 @@ -0,0 +1,19 @@ +#!/bin/bash + +function die { + echo 1>&2 "${1}" + exit 1 +} + +if [[ $(whoami) == "root" ]] ; then + su - postgres ${0} ${@} + exit $? +fi + +if [[ $(whoami) != "postgres" ]] ; then + die "${0} : must be run as postgres user" +fi + +cd ~bubble/sql \ + && init_bubble_db.sh {{ db_name }} {{ db_user }} {{ is_fork }} {{ install_type }} ${1} \ + || die "error reinitializing database" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/templates/refresh_bubble_ssh_keys.sh.j2 b/bubble-server/src/main/resources/ansible/roles/bubble/templates/refresh_bubble_ssh_keys.sh.j2 new file mode 100644 index 00000000..ba9eae5a --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/templates/refresh_bubble_ssh_keys.sh.j2 @@ -0,0 +1,65 @@ +#!/bin/bash + +LOG=/tmp/bubble.refresh_bubble_ssh_keys.log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "$(date): ${1}" >> ${LOG} +} + +CURRENT_KEYS_SQL=' +SELECT k.ssh_public_key +FROM account_ssh_key k, account a +WHERE a.uuid = k.account + AND a.admin = true + AND k.install_ssh_key = true + AND (k.expiration is null or k.expiration < 1000*extract(epoch from now()))' + +AUTH_KEYS="/root/.ssh/authorized_keys" + +NEW_KEYS=$(mktemp /root/.ssh/authorized_keys.XXXXXXX) +chmod 600 ${NEW_KEYS} || die "Error setting permissions on new authorized_keys file: ${NEW_KEYS}" + +KEY_COUNT=0 +for key in $(echo "${CURRENT_KEYS_SQL}" | PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble -qt) ; do + if [[ -z "$(echo "${key}" | tr -d [[:space:]])" ]] ; then + continue + fi + KEY="$(bdecrypt "${key}" 2> /dev/null)" + if [[ ! -z "${KEY}" && "${KEY}" == ssh-rsa* ]] ; then + log "Adding authorized key: $(echo "${KEY}" | tr -d '\n')" + echo "${KEY}" >> ${NEW_KEYS} + KEY_COUNT=$(expr ${KEY_COUNT} + 1) + else + log "Warning: NOT adding malformed key: $(echo "${KEY}" | tr -d '\n')" + fi +done + +if [[ ${KEY_COUNT} -eq 0 ]] ; then + # Sanity check that we can even talk to psql + # We may be out of memory, in which case we do not want to erase existing installed keys + if [[ -z "$(echo 'SELECT count(*) FROM account_ssh_key' | PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble)" ]] ; then + log "Warning: error calling psql, not installing/uninstalling any keys" + exit 0 + fi +fi + +# Retain self-generated ansible setup key +ANSIBLE_USER="{{admin_user}}" +if [[ ! -z "${ANSIBLE_USER}" ]] ; then + PUB_FILE="$(cd ~{{admin_user}} && pwd)/.ssh/bubble_rsa.pub" + if [[ -f "${PUB_FILE}" ]] ; then + cat "${PUB_FILE}" >> ${NEW_KEYS} + fi +else + log "Warning: No ansible user defined, unable to retain setup key" +fi + +mv ${NEW_KEYS} ${AUTH_KEYS} || die "Error moving ${NEW_KEYS} -> ${AUTH_KEYS}" + +log "Installed ${KEY_COUNT} authorized SSH keys: ${NEW_KEYS} -> ${AUTH_KEYS}" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/templates/snapshot_ansible.sh.j2 b/bubble-server/src/main/resources/ansible/roles/bubble/templates/snapshot_ansible.sh.j2 new file mode 100644 index 00000000..635fcb9c --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble/templates/snapshot_ansible.sh.j2 @@ -0,0 +1,34 @@ +#!/bin/bash + +SCRIPT="${0}" +SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) + +LOG=/tmp/$(basename ${0}).log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "${1}" | tee -a ${LOG} +} + +if [[ $(whoami) != "{{ admin_user }}" ]] ; then + if [[ $(whoami) == "root" ]] ; then + sudo -H -u "{{ admin_user }}" ${0} + exit $? + fi + die "${0} must be run as {{ admin_user }}" +fi + +ANSIBLE_USER_HOME=$(cd ~{{ admin_user }} && pwd) + +ANSIBLE_SNAPSHOT="/home/bubble/ansible.tgz" + +cd ${ANSIBLE_USER_HOME} \ + && tar czf ${ANSIBLE_SNAPSHOT} ./ansible \ + && chmod 400 ${ANSIBLE_SNAPSHOT} \ + && chown bubble ${ANSIBLE_SNAPSHOT} \ + || die "Error creating ansible snapshot" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json new file mode 100644 index 00000000..7870da24 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json @@ -0,0 +1,10 @@ +{ + "name": "bubble_finalizer", + "config": [ + {"name": "server_alias", "value": "[[network.networkDomain]]"}, + {"name": "restore_key", "value": "[[restoreKey]]"}, + {"name": "install_type", "value": "[[installType]]"}, + {"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[expr nodeSize.memoryMB '//' '2.625']]m"} + ], + "optionalConfigNames": ["restore_key"] +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/tasks/main.yml new file mode 100644 index 00000000..a7262036 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/tasks/main.yml @@ -0,0 +1,38 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Snapshot ansible roles + shell: snapshot_ansible.sh + +- name: Touch first-time setup file + shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi" + when: restore_key is not defined + +- name: Touch first-time setup file (restore) + shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n restore > /home/bubble/first_time_marker ; fi" + when: restore_key is defined + +- name: Install mitmproxy CA cert in local CA store + shell: install_cert.sh /home/mitmproxy/.mitmproxy/{{ server_alias }}-ca-cert.pem 600 + when: install_type == 'node' + +- name: Install mitmproxy public certs in bubble dir + shell: /usr/local/bin/copy_certs_to_bubble.sh {{ server_alias }} + when: install_type == 'node' + +- name: Install bubble supervisor conf file + template: + src: "supervisor_bubble.conf.j2" + dest: /etc/supervisor/conf.d/bubble.conf + +# We cannot receive notifications until nginx is running, so start bubble API as the very last step +- name: Ensure bubble and bubble_nodemanager are started + supervisorctl: + name: '{{ item }}' + state: restarted + with_items: + - bubble + - nodemanager + +- name: Ensure authorized SSH keys are up-to-date + shell: su - bubble bash -c "touch /home/bubble/.refresh_ssh_keys" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/templates/supervisor_bubble.conf.j2 b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/templates/supervisor_bubble.conf.j2 new file mode 100644 index 00000000..55b2281e --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/templates/supervisor_bubble.conf.j2 @@ -0,0 +1,10 @@ + +[program:bubble] +stdout_logfile = /home/bubble/logs/bubble-out.log +stderr_logfile = /home/bubble/logs/bubble-err.log +command=sudo -u bubble bash -c "/usr/bin/java \ + -Dfile.encoding=UTF-8 -Djava.net.preferIPv4Stack=true \ + -XX:+UseG1GC -XX:MaxGCPauseMillis=400 {{ bubble_java_opts }} \ + -cp /home/bubble/current/bubble.jar \ + bubble.server.BubbleServer \ + /home/bubble/current/bubble.env" diff --git a/bubble-server/src/main/resources/ansible/roles/common/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/common/files/bubble_role.json new file mode 100644 index 00000000..8ed81078 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/common/files/bubble_role.json @@ -0,0 +1,6 @@ +{ + "name": "common", + "config": [ + {"name": "hostname", "value": "[[node.fqdn]]"} + ] +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/common/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/common/tasks/main.yml new file mode 100644 index 00000000..2e29cb67 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/common/tasks/main.yml @@ -0,0 +1,6 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Set hostname to {{ hostname }} + hostname: + name: '{{ hostname }}' diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/nginx/files/bubble_role.json new file mode 100644 index 00000000..59055078 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/files/bubble_role.json @@ -0,0 +1,11 @@ +{ + "name": "nginx", + "config": [ + {"name": "server_name", "value": "[[node.fqdn]]"}, + {"name": "server_alias", "value": "[[network.networkDomain]]"}, + {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, + {"name": "ssl_port", "value": "[[node.sslPort]]"}, + {"name": "admin_port", "value": "[[node.adminPort]]"}, + {"name": "install_type", "value": "[[installType]]"} + ] +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml new file mode 100644 index 00000000..e64f29aa --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml @@ -0,0 +1,22 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Init certbot + shell: init_certbot.sh {{ letsencrypt_email }} {{ server_name }} {{ server_alias }} + +# see https://weakdh.org/sysadmin.html +- name: Create a strong dhparam.pem + shell: openssl dhparam -out /etc/nginx/dhparams.pem 2048 + args: + creates: /etc/nginx/dhparams.pem + +- name: Create dhparam nginx conf + template: src=stronger_dhparams.conf dest=/etc/nginx/conf.d/stronger_dhparams.conf + +- include: site.yml +- meta: flush_handlers # nginx has to be restarted right now if it has to + +- name: Ensure nginx is restarted + service: + name: nginx + state: restarted diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/tasks/site.yml b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/site.yml new file mode 100644 index 00000000..775ffb3b --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/site.yml @@ -0,0 +1,40 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Disable default site + file: + path: /etc/nginx/sites-enabled/default + state: absent + +- name: Create the .well-known directory + file: + path: /var/www/html/.well-known + owner: www-data + group: www-data + state: directory + +- name: Create default nginx site (type={{ install_type }}) + template: + src: "site_{{ install_type }}.conf.j2" + dest: "/etc/nginx/sites-available/{{ server_name }}.conf" + +- name: Create alias nginx site (type={{ install_type }}) + template: + src: "site_{{ install_type }}_alias.conf.j2" + dest: "/etc/nginx/sites-available/{{ server_alias }}.conf" + +- name: Symlink default site to site-enabled + file: + src: /etc/nginx/sites-available/{{ server_name }}.conf + dest: /etc/nginx/sites-enabled/{{ server_name }}.conf + owner: root + group: root + state: link + +- name: Symlink alias site to site-enabled + file: + src: /etc/nginx/sites-available/{{ server_alias }}.conf + dest: /etc/nginx/sites-enabled/{{ server_alias }}.conf + owner: root + group: root + state: link diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node.conf.j2 b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node.conf.j2 new file mode 100644 index 00000000..90b457a5 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node.conf.j2 @@ -0,0 +1,54 @@ +server { + server_name {{ server_name }}; + listen 80; + listen 1080; + listen {{ ssl_port }} ssl http2; + listen 443 ssl http2; + client_max_body_size 200M; + + root /home/bubble/site/; + index index.html; + + location / { + rewrite ^/(\w+/)+(?.*)$ /$basename break; + try_files $uri /index.html =404; + } + + location /api { + proxy_pass http://127.0.0.1:{{ admin_port }}/api; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location /nodeman { + return 302 /nodeman/; + } + location /nodeman/ { + proxy_pass http://127.0.0.1:7800/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location ^~ /.well-known/acme-challenge/ { + default_type "text/plain"; + root /var/www/html; + } + + ssl_certificate /etc/letsencrypt/live/{{ server_name }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ server_name }}/privkey.pem; + ssl_session_cache shared:le_nginx_SSL:1m; + ssl_session_timeout 1440m; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; + + if ($scheme != "https") { + return 301 https://$host:{{ ssl_port }}$request_uri; + } +} diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node_alias.conf.j2 b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node_alias.conf.j2 new file mode 100644 index 00000000..40dcc06c --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node_alias.conf.j2 @@ -0,0 +1,54 @@ +server { + server_name {{ server_alias }}; + listen 80; + listen 1080; + listen {{ ssl_port }} ssl http2; + listen 443 ssl http2; + client_max_body_size 200M; + + root /home/bubble/site/; + index index.html; + + location / { + rewrite ^/(\w+/)+(?.*)$ /$basename break; + try_files $uri /index.html =404; + } + + location /api { + proxy_pass http://127.0.0.1:{{ admin_port }}/api; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location /nodeman { + return 302 /nodeman/; + } + location /nodeman/ { + proxy_pass http://127.0.0.1:7800/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location ^~ /.well-known/acme-challenge/ { + default_type "text/plain"; + root /var/www/html; + } + + ssl_certificate /etc/letsencrypt/live/{{ server_alias }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ server_alias }}/privkey.pem; + ssl_session_cache shared:le_nginx_SSL:1m; + ssl_session_timeout 1440m; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; + + if ($scheme != "https") { + return 301 https://$host:{{ ssl_port }}$request_uri; + } +} diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage.conf.j2 b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage.conf.j2 new file mode 100644 index 00000000..98892225 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage.conf.j2 @@ -0,0 +1,52 @@ +server { + listen 80; + server_name {{ server_name }}; + client_max_body_size 200M; + + root /home/bubble/site/; + index index.html; + + location / { + rewrite ^/(\w+/)+(?.*)$ /$basename break; + try_files $uri /index.html =404; + } + + location /api { + proxy_pass http://127.0.0.1:{{ admin_port }}/api; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location /nodeman { + return 302 /nodeman/; + } + location /nodeman/ { + proxy_pass http://127.0.0.1:7800/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location ^~ /.well-known/acme-challenge/ { + default_type "text/plain"; + root /var/www/html; + } + + listen 443 ssl; + ssl_certificate /etc/letsencrypt/live/{{ server_name }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ server_name }}/privkey.pem; + ssl_session_cache shared:le_nginx_SSL:1m; + ssl_session_timeout 1440m; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; + + if ($scheme != "https") { + return 301 https://$host$request_uri; + } +} diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage_alias.conf.j2 b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage_alias.conf.j2 new file mode 100644 index 00000000..e2802afe --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage_alias.conf.j2 @@ -0,0 +1,52 @@ +server { + listen 80; + server_name {{ server_alias }}; + client_max_body_size 200M; + + root /home/bubble/site/; + index index.html; + + location / { + rewrite ^/(\w+/)+(?.*)$ /$basename break; + try_files $uri /index.html =404; + } + + location /api { + proxy_pass http://127.0.0.1:{{ admin_port }}/api; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location /nodeman { + return 302 /nodeman/; + } + location /nodeman/ { + proxy_pass http://127.0.0.1:7800/; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Host {{ server_name }}; + proxy_set_header X-Forwarded-Proto https; + } + + location ^~ /.well-known/acme-challenge/ { + default_type "text/plain"; + root /var/www/html; + } + + listen 443 ssl; + ssl_certificate /etc/letsencrypt/live/{{ server_alias }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ server_alias }}/privkey.pem; + ssl_session_cache shared:le_nginx_SSL:1m; + ssl_session_timeout 1440m; + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + + ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA ECDHE-ECDSA-AES256-SHA ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-SHA DHE-RSA-AES256-SHA DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256 EDH-RSA-DES-CBC3-SHA"; + + if ($scheme != "https") { + return 301 https://$host$request_uri; + } +} diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/templates/stronger_dhparams.conf b/bubble-server/src/main/resources/ansible/roles/nginx/templates/stronger_dhparams.conf new file mode 100644 index 00000000..580e8c9e --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/nginx/templates/stronger_dhparams.conf @@ -0,0 +1 @@ +ssl_dhparam /etc/nginx/dhparams.pem; diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index ed11e97e..a0cef4b2 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -256,6 +256,19 @@ {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50}, {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} ], + "packer": { + "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "credentials.apiKey"}], + "builder": { + "type": "digitalocean", + "ssh_username": "root", + "api_token": "<>", + "image": "ubuntu-18-04-x64", + "region": "[[region]]", + "size": "s-1vcpu-1gb", + "ipv6": true, + "tags": ["packer-bubble"] + } + }, "config": [{"name": "os", "value": "ubuntu-18-04-x64"}] }, "credentials": { diff --git a/bubble-server/src/main/resources/packer/hosts b/bubble-server/src/main/resources/packer/hosts new file mode 100644 index 00000000..975be3a8 --- /dev/null +++ b/bubble-server/src/main/resources/packer/hosts @@ -0,0 +1,2 @@ +[bubble] +127.0.0.1 ansible_python_interpreter=/usr/bin/python3 \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/packer-sage-playbook.yml b/bubble-server/src/main/resources/packer/packer-sage-playbook.yml new file mode 100644 index 00000000..14807d9c --- /dev/null +++ b/bubble-server/src/main/resources/packer/packer-sage-playbook.yml @@ -0,0 +1,25 @@ +--- + +- name: Create new bubble sage node + hosts: bubble + remote_user: root + gather_facts: no + vars: + install_type: sage + + pre_tasks: + - name: apt install python3 and python3-pip + raw: sudo apt-get -y install python3 python3-pip virtualenv + + - name: pip install setuptools and psycopg2-binary + raw: sudo pip3 install setuptools psycopg2-binary + + - name: gather facts + setup: + + roles: + - common + - firewall + - nginx + - bubble + - bubble_finalizer diff --git a/bubble-server/src/main/resources/packer/packer-sage.json.hbs b/bubble-server/src/main/resources/packer/packer-sage.json.hbs new file mode 100644 index 00000000..0efca0cd --- /dev/null +++ b/bubble-server/src/main/resources/packer/packer-sage.json.hbs @@ -0,0 +1,34 @@ +{ + "variables": { + [[#each packer.vars]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]], +[[/unless]][[/each]] + }, + "builders": [ + [[#each packer.builders]][[json this]][[#unless @last]], +[[/unless]][[/each]] +], + "provisioners": [ + { + "type": "shell", + "inline": [ + "sleep 30", + "sudo apt-get -y update", + "sudo apt-get -y upgrade", + "sudo apt-get -y install python3 python3-pip virtualenv", + "sudo pip3 install setuptools psycopg2-binary ansible" + ] + }, + { + "type": "ansible-local", + "playbook_file": "packer-sage-playbook.yml", + "role_paths": ["."], + "inventory_file": "hosts" + } + ], + "post-processors": [ + { + "type": "manifest", + "output": "manifest.json" + } + ] +} diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/bsql.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/bsql.sh new file mode 100755 index 00000000..03a84a98 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/bsql.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +PGPASSWORD="$(cat /home/bubble/.BUBBLE_PG_PASSWORD)" psql -U bubble -h 127.0.0.1 bubble "${@}" diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh new file mode 100755 index 00000000..490fb769 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +BUBBLE_HOME="/home/bubble" +RESTORE_MARKER="${BUBBLE_HOME}/.restore" +RESTORE_RUN_MARKER="${BUBBLE_HOME}/.restore_run" + +SELF_NODE="self_node.json" +BUBBLE_SELF_NODE="${BUBBLE_HOME}/${SELF_NODE}" + +ADMIN_PORT=${1:?no admin port provided} +TIMEOUT=${2:-3600} # 60 minutes default timeout + +LOG=/tmp/bubble.restore.log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "${1}" >> ${LOG} +} + +START=$(date +%s) +while [[ ! -f "${RESTORE_MARKER}" ]] ; do + sleep 5 + if [[ $(expr $(date +%s) - ${START}) -gt ${TIMEOUT} ]] ; then + break + fi +done + +if [[ ! -f "${RESTORE_MARKER}" ]] ; then + die "Restore marker was never created: ${RESTORE_MARKER}" +fi + +# was a restore already attempted? only one attempt is allowed. start another restore (with a new node) if you need to try again +if [[ -f ${RESTORE_RUN_MARKER} ]] ; then + die "Restore was already attempted, cannot attempt again" +fi +touch ${RESTORE_RUN_MARKER} + +# Ensure there is only one self_node.json in the backup. Otherwise maybe we have more than once backup, can't restore. +SELF_NODE_COUNT=$(find ${BUBBLE_HOME}/restore -type f -name "${SELF_NODE}" | wc -l | tr -d ' ') +if [[ ${SELF_NODE_COUNT} -eq 0 ]] ; then + die "Cannot restore, restore base could not be determined (no ${SELF_NODE} found under ${BUBBLE_HOME}/restore)" +elif [[ ${SELF_NODE_COUNT} -gt 1 ]] ; then + die "Cannot restore, restore base could not be determined (multiple ${SELF_NODE} files found under ${BUBBLE_HOME}/restore): $(find ${BUBBLE_HOME}/restore -type f -name "${SELF_NODE}")" +fi + +# set RESTORE_BASE, ensure it is set +RESTORE_BASE=$(dirname $(find ${BUBBLE_HOME}/restore -type f -name "${SELF_NODE}" | head -1)) +if [[ -z "${RESTORE_BASE}" ]] ; then + die "Cannot restore, restore base could not be determined (no ${SELF_NODE} found under ${BUBBLE_HOME}/restore)" +fi + +# stop bubble service +log "Stopping bubble service" +supervisorctl stop bubble + +# stop mitmdump service +log "Stopping mitmproxy service" +supervisorctl stop mitmdump + +# restore bubble.jar +log "Restoring bubble.jar" +cp ${RESTORE_BASE}/bubble.jar ${BUBBLE_HOME}/current/bubble.jar + +# set wasRestored flag in self_node.json +log "Adding wasRestored=true to ${SELF_NODE}" +TEMP_SELF=$(mktemp /tmp/self_node.XXXXXXX.json) +cat ${BUBBLE_SELF_NODE} | jq '.wasRestored = true' > ${TEMP_SELF} || die "Error adding 'wasRestored' flag to ${SELF_NODE}" +cat ${TEMP_SELF} > ${BUBBLE_SELF_NODE} || die "Error rewriting ${SELF_NODE}" + +log "Setting ownership of json files to bubble user" +chown bubble ${BUBBLE_HOME}/*.json || die "Error changing ownership of json files to bubble user" + +# restore dot files +log "Restoring bubble dotfiles" +cp ${RESTORE_BASE}/dotfiles/.BUBBLE_* ${BUBBLE_HOME}/ || die "Error restoring dotfiles" + +# restore mitm configs +log "Restoring mitm certs" +cp -R ${RESTORE_BASE}/mitm_certs ${BUBBLE_HOME}/ || die "Error restoring mitm certs" + +# drop and recreate database from backup (but preserve bubble_node and bubble_node_key for current node) +log "Restoring bubble database" +cp ${RESTORE_BASE}/bubble.sql.gz ${BUBBLE_HOME}/sql/ \ + && chown -R bubble ${BUBBLE_HOME}/sql \ + && chgrp -R postgres ${BUBBLE_HOME}/sql \ + && chmod 550 ${BUBBLE_HOME}/sql \ + && chmod 440 ${BUBBLE_HOME}/sql/* || die "Error restoring bubble database archive" +su - postgres bash -c "cd ${BUBBLE_HOME}/sql && full_reset_db.sh drop" || die "Error restoring database" + +# Remove old keys +log "Removing node keys" +echo "DELETE FROM bubble_node_key" | bsql.sh + +# restore local storage +log "Restoring bubble LocalStorage" +rm -rf ${BUBBLE_HOME}/.bubble_local_storage/* && rsync -ac ${RESTORE_BASE}/LocalStorage/* ${BUBBLE_HOME}/.bubble_local_storage/ || die "Error restoring LocalStorage" + +# flush redis +log "Flushing redis" +echo "FLUSHALL" | redis-cli || die "Error flushing redis" + +# restore algo configs +CONFIGS_BACKUP=/home/bubble/.BUBBLE_ALGO_CONFIGS.tgz +if [[ ! -f ${CONFIGS_BACKUP} ]] ; then + log "Warning: Algo VPN configs backup not found: ${CONFIGS_BACKUP}, not installing algo" +else + ALGO_BASE=/root/ansible/roles/algo/algo + if [[ ! -d ${ALGO_BASE} ]] ; then + die "Error restoring Algo VPN: directory ${ALGO_BASE} not found" + fi + cd ${ALGO_BASE} && tar xzf ${CONFIGS_BACKUP} || die "Error restoring algo VPN configs" + + # install/configure algo + ${ALGO_BASE}/install_algo.sh || die "Error configuring or installing algo VPN" + + # ensure user monitor is running + supervisorctl restart algo_refresh_users_monitor +fi + +# restart mitm proxy service +log "Restarting mitmproxy" +supervisorctl restart mitmdump + +# restart bubble service +log "Restore complete: restarting bubble API" +supervisorctl restart bubble + +# verify service is running OK +log "Pausing for a bit, then verifying bubble server has successfully restarted after restore" +sleep 60 +curl https://$(hostname):${ADMIN_PORT}/api/.bubble || log "Error restarting bubble server" + +# remove restore markers, we are done +log "Cleaning up temp files" +rm -f ${RESTORE_MARKER} ${RESTORE_RUN_MARKER} diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh new file mode 100644 index 00000000..3e6467a1 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +echo "$@" > /tmp/init.args + +LOG=/dev/null + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "${1}" >> ${LOG} +} + +export LANG="en_US.UTF-8" +export LANGUAGE="en_US.UTF-8" +export LC_CTYPE="en_US.UTF-8" +export LC_NUMERIC="en_US.UTF-8" +export LC_TIME="en_US.UTF-8" +export LC_COLLATE="en_US.UTF-8" +export LC_MONETARY="en_US.UTF-8" +export LC_MESSAGES="en_US.UTF-8" +export LC_PAPER="en_US.UTF-8" +export LC_NAME="en_US.UTF-8" +export LC_ADDRESS="en_US.UTF-8" +export LC_TELEPHONE="en_US.UTF-8" +export LC_MEASUREMENT="en_US.UTF-8" +export LC_IDENTIFICATION="en_US.UTF-8" +export LC_ALL=en_US.UTF-8 + +if [[ "$(whoami)" != "postgres" ]] ; then + echo "Must be run as postgres user" + exit 1 +fi + +DB_NAME=${1:?no db name provided} +DB_USER=${2:?no db user provided} +IS_FORK=${3:?no fork argument provided} +INSTALL_MODE=${4:?no install mode provided} +DROP_AND_RECREATE=${5} + +BUBBLE_HOME=/home/bubble +BUBBLE_JAR=/home/bubble/current/bubble.jar +if [[ ! -f ${BUBBLE_JAR} ]] ; then + die "Bubble jar not found: ${BUBBLE_JAR}" +fi + +function user_exists { + username="${1}" + num_users="$(echo "select count(*) from pg_user where usename='${username}'" | psql -qt | egrep -v '^$')" + if [[ -z "${num_users}" || ${num_users} -eq 0 ]] ; then + echo "0" + else + echo "1" + fi +} + +function db_exists { + dbname="${1}" + num_dbs="$(echo "select count(*) from pg_database where datname='${dbname}'" | psql -qt | egrep -v '^$')" + if [[ -z "${num_dbs}" || ${num_dbs} -eq 0 ]] ; then + echo "0" + else + echo "1" + fi +} + +function count_table_rows { + dbname="${1}" + tname="${2}" + num_rows="$(echo "select count(*) from ${tname}" | psql -qt ${dbname} | egrep -v '^$')" + if [[ -z "${num_rows}" ]] ; then + die "count_table_rows: error counting rows for table ${tname}" + fi + echo ${num_rows} +} + +if [[ ! -z "${DROP_AND_RECREATE}" && "${DROP_AND_RECREATE}" == "drop" ]] ; then + dropdb ${DB_NAME} || echo "error dropping DB ${DB_NAME} (will continue)" + dropuser ${DB_USER} || echo "error dropping DB user ${DB_USER} (will continue)" + uuid > ${BUBBLE_HOME}/.BUBBLE_PG_PASSWORD +fi + +if [[ $(user_exists ${DB_USER}) -eq 0 ]] ; then + log "Creating user ${DB_USER}" + if [[ "$(echo ${IS_FORK} | tr [[:upper:]] [[:lower:]])" == "true" ]] ; then + createuser --createdb --no-createrole --no-superuser --no-replication ${DB_USER} || die "Error creating user" + else + createuser --no-createdb --no-createrole --no-superuser --no-replication ${DB_USER} || die "Error creating user" + fi + DB_PASS="$(cat ${BUBBLE_HOME}/.BUBBLE_PG_PASSWORD)" + echo "ALTER USER bubble WITH PASSWORD '${DB_PASS}'" | psql || die "Error setting user password" +fi + +if [[ $(db_exists ${DB_NAME}) -eq 0 ]] ; then + log "Creating DB ${DB_NAME}" + createdb --encoding=UTF-8 ${DB_NAME} || die "Error creating DB" +fi + +if [[ $(count_table_rows ${DB_NAME} account 2> /dev/null) -eq 0 ]] ; then + TEMP_DB="${DB_NAME}_$(uuid | tr -d '-')" + log "Creating tempDB ${TEMP_DB}" + createdb --encoding=UTF-8 ${TEMP_DB} || die "Error creating temp DB" + log "Populating tempDB ${TEMP_DB} with bubble.sql.gz" + zcat /home/bubble/sql/bubble.sql.gz | psql ${TEMP_DB} || die "Error writing database schema/data" + DB_KEY="$(cat ${BUBBLE_HOME}/.BUBBLE_DB_ENCRYPTION_KEY)" + TO_KEY="$(uuid)" + if [[ -z "${TO_KEY}" ]] ; then + dropdb ${TEMP_DB} + die "${BUBBLE_HOME}/.BUBBLE_DB_ENCRYPTION_KEY does not exist or is empty" + fi + log "Dumping schema from ${TEMP_DB} -> ${DB_NAME}" + pg_dump --schema-only ${TEMP_DB} | psql ${DB_NAME} + # log "Rekeying: fromKey=${DB_KEY}, toKey=${TO_KEY}" + java -cp ${BUBBLE_JAR} bubble.main.RekeyDatabaseMain \ + --jar ${BUBBLE_JAR} \ + --db-user ${DB_USER} \ + --db-password "${DB_PASS}" \ + --from-db ${TEMP_DB} \ + --from-key "${DB_KEY}" \ + --to-db ${DB_NAME} \ + --to-key "${TO_KEY}" 2>&1 || (dropdb ${TEMP_DB} ; die "Error re-keying database") +# --to-key "${TO_KEY}" 2>&1 | tee -a ${LOG} || (dropdb ${TEMP_DB} ; die "Error re-keying database") + log "Rekey successful, dropping ${TEMP_DB}" + dropdb ${TEMP_DB} + log "Saving ${TO_KEY} to ${BUBBLE_HOME}/.BUBBLE_DB_ENCRYPTION_KEY" + echo -n "${TO_KEY}" > ${BUBBLE_HOME}/.BUBBLE_DB_ENCRYPTION_KEY +fi + +echo "DELETE FROM bubble_node_key WHERE node IN (SELECT uuid FROM bubble_node WHERE ip4='127.0.0.1' OR ip4='' OR ip4 IS NULL)" | psql ${DB_NAME} \ + || die "Error removing bubble_node_keys with remote_host=127.0.0.1" +echo "DELETE FROM bubble_node WHERE ip4='127.0.0.1'" | psql ${DB_NAME} \ + || die "Error removing bubble_nodes with ip4=127.0.0.1" + +if [[ "${INSTALL_MODE}" == "node" ]] ; then + echo "UPDATE account SET locked=true" | psql ${DB_NAME} \ + || die "Error locking accounts" +fi diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/init_roles.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/init_roles.sh new file mode 100755 index 00000000..6a56c15a --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/init_roles.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +SCRIPT="${0}" +SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) + +LOG=/tmp/$(basename ${0}).log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "${1}" | tee -a ${LOG} +} + +if [[ $(whoami) != "bubble" ]] ; then + if [[ $(whoami) == "root" ]] ; then + sudo -H -u bubble ${0} + exit $? + fi + die "${0} must be run as bubble" +fi + +if [[ -z "${LOCALSTORAGE_BASE_DIR}" ]] ; then + if [[ -f "${HOME}/bubble/current/bubble.env" ]] ; then + LOCALSTORAGE_BASE_DIR=$(cat "${HOME}/bubble/current/bubble.env" | grep -v '^#' | grep LOCALSTORAGE_BASE_DIR | awk -F '=' '{print $2}' | tr -d ' ') + fi +fi +if [[ -z "${LOCALSTORAGE_BASE_DIR}" ]] ; then + log "LOCALSTORAGE_BASE_DIR env var not defined, using ${HOME}/.bubble_local_storage" + LOCALSTORAGE_BASE_DIR="${HOME}/.bubble_local_storage" +fi + +if [[ -z "${BUBBLE_JAR}" ]] ; then + if [[ -f "${HOME}/current/bubble.jar" ]] ; then + BUBBLE_JAR="${HOME}/current/bubble.jar" + fi +fi +if [[ -z "${BUBBLE_JAR}" ]] ; then + die "BUBBLE_JAR env var not set and no jar file found" +fi + +ROLE_DIR="${HOME}/role_tgz" +if [[ ! -d "${ROLE_DIR}" ]] ; then + die "role_tgz dir not found: ${ROLE_DIR}" +fi + +NETWORK_UUID="$(cat ${HOME}/self_node.json | jq -r .network)" +find ${ROLE_DIR} -type f -name "*.tgz" | while read role_tgz ; do + path="automation/roles/$(basename ${role_tgz})" + dest="${LOCALSTORAGE_BASE_DIR}/${NETWORK_UUID}/${path}" + if [[ ! -f ${dest} ]] ; then + mkdir -p $(dirname ${dest}) || die "Error creating destination directory" + cp ${role_tgz} ${dest} || die "Error copying role archive" + log "installed role ${role_tgz} -> ${dest}" + else + log "role already installed ${role_tgz} -> ${dest}" + fi +done diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/pg_hba.conf b/bubble-server/src/main/resources/packer/roles/bubble/files/pg_hba.conf new file mode 100644 index 00000000..b708a885 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/pg_hba.conf @@ -0,0 +1,99 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# plain TCP/IP socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", +# "ident", "peer", "pam", "ldap", "radius" or "cert". Note that +# "password" sends passwords in clear text; "md5" is preferred since +# it sends encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + + + + +# DO NOT DISABLE! +# If you change this first entry you will need to make sure that the +# database superuser can access the database using some other method. +# Noninteractive access to all databases is required during automatic +# maintenance (custom daily cronjobs, replication, and similar tasks). +# +# Database administrative login by Unix domain socket +local all postgres peer + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 md5 +# IPv6 local connections: +host all all ::1/128 md5 +# Allow replication connections from localhost, by a user with the +# replication privilege. +#local replication postgres peer +#host replication postgres 127.0.0.1/32 md5 +#host replication postgres ::1/128 md5 diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/postgresql.conf b/bubble-server/src/main/resources/packer/roles/bubble/files/postgresql.conf new file mode 100644 index 00000000..aa731515 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/postgresql.conf @@ -0,0 +1,614 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, or use "pg_ctl reload". Some +# parameters, which are marked below, require a server shutdown and restart to +# take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: kB = kilobytes Time units: ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +data_directory = '/var/lib/postgresql/9.4/main' # use data in another directory + # (change requires restart) +hba_file = '/etc/postgresql/9.4/main/pg_hba.conf' # host-based authentication file + # (change requires restart) +ident_file = '/etc/postgresql/9.4/main/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +external_pid_file = '/var/run/postgresql/9.4-main.pid' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = 'localhost,127.0.0.1' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +port = 5432 # (change requires restart) +max_connections = 197 # (change requires restart) +# Note: Increasing max_connections costs ~400 bytes of shared memory per +# connection slot, plus lock space (see max_locks_per_transaction). +superuser_reserved_connections = 3 # (change requires restart) +unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - Security and Authentication - + +#authentication_timeout = 1min # 1s-600s +ssl = true # (change requires restart) +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers + # (change requires restart) +#ssl_prefer_server_ciphers = on # (change requires restart) +#ssl_ecdh_curve = 'prime256v1' # (change requires restart) +#ssl_renegotiation_limit = 512MB # amount of data between renegotiations +ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' # (change requires restart) +ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' # (change requires restart) +#ssl_ca_file = '' # (change requires restart) +#ssl_crl_file = '' # (change requires restart) +#password_encryption = on +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = '' +#krb_caseins_users = off + +# - TCP Keepalives - +# see "man 7 tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Note: Increasing max_prepared_transactions costs ~600 bytes of shared memory +# per transaction slot, plus lock space (see max_locks_per_transaction). +# It is not advisable to set max_prepared_transactions nonzero unless you +# actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#max_stack_depth = 2MB # min 100kB +dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # use none to disable dynamic shared memory + +# - Disk - + +#temp_file_limit = -1 # limits per-session temp file space + # in kB, or -1 for no limit + +# - Kernel Resource Usage - + +#max_files_per_process = 1000 # min 25 + # (change requires restart) +#shared_preload_libraries = '' # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # 0-1000 max buffers written/round +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multipler on buffers scanned/round + +# - Asynchronous Behavior - + +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 + + +#------------------------------------------------------------------------------ +# WRITE AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = minimal # minimal, archive, hot_standby, or logical + # (change requires restart) +#fsync = on # turns forced synchronization on or off +#synchronous_commit = on # synchronization level; + # off, local, remote_write, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_segments = 3 # in logfile segments, min 1, 16MB each +#checkpoint_timeout = 5min # range 30s-1h +#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_warning = 30s # 0 disables + +# - Archiving - + +#archive_mode = off # allows archiving to be done + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Server(s) - + +# Set these on the master and on any standby that will send replication data. + +#max_wal_senders = 0 # max number of walsender processes + # (change requires restart) +#wal_keep_segments = 0 # in logfile segments, 16MB each; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables + +#max_replication_slots = 0 # max number of replication slots + # (change requires restart) + +# - Master Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a master server. + +#hot_standby = off # "on" allows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from master + # in milliseconds; 0 disables + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#effective_cache_size = 4GB + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses + + +#------------------------------------------------------------------------------ +# ERROR REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'pg_log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' + +# This is only relevant when logging to eventlog (win32): +#event_source = 'PostgreSQL' + +# - When to Log - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +log_line_prefix = '%t [%p-%l] %q%u@%d ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %p = process ID + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_statement = 'none' # none, ddl, mod, all +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'localtime' + + +#------------------------------------------------------------------------------ +# RUNTIME STATISTICS +#------------------------------------------------------------------------------ + +# - Query/Index Statistics Collector - + +#track_activities = on +#track_counts = on +#track_io_timing = off +#track_functions = none # none, pl, all +#track_activity_query_size = 1024 # (change requires restart) +#update_process_title = on +stats_temp_directory = '/var/run/postgresql/9.4-main.pg_stat_tmp' + + +# - Statistics Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM PARAMETERS +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#search_path = '"$user",public' # schema names +#default_tablespace = '' # a tablespace name, '' uses the default +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_min_age = 50000000 +#vacuum_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_freeze_table_age = 150000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' + +# - Locale and Formatting - + +datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'localtime' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 0 # min -15, max 3 +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.UTF-8' # locale for system error message + # strings +lc_monetary = 'en_US.UTF-8' # locale for monetary formatting +lc_numeric = 'en_US.UTF-8' # locale for number formatting +lc_time = 'en_US.UTF-8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#local_preload_libraries = '' +#session_preload_libraries = '' + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +# Note: Each lock table slot uses ~270 bytes of shared memory, and there are +# max_locks_per_transaction * (max_connections + max_prepared_transactions) +# lock table slots. +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) + + +#------------------------------------------------------------------------------ +# VERSION/PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#default_with_oids = off +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#sql_inheritance = on +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. + +#include_dir = 'conf.d' # include files ending in '.conf' from + # directory 'conf.d' +#include_if_exists = 'exists.conf' # include file only if it exists +#include = 'special.conf' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/random_password.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/random_password.sh new file mode 100755 index 00000000..2a28f048 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/random_password.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +file=${1:?no file provided} +owner=${2:?no owner provided} +group=${3:?no group provided} + +if [[ ! -f ${file} ]] ; then + touch ${file} && chmod 660 ${file} && chown ${owner} ${file} && chgrp ${group} ${file} && uuid | tr -d '\n' > ${file} +fi diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys_monitor.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys_monitor.sh new file mode 100644 index 00000000..8a61af69 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys_monitor.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +LOG=/tmp/bubble.ssh_keys_monitor.log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "$(date): ${1}" >> ${LOG} +} + +SSH_KEY_BASE=/root/.ssh +if [[ ! -d ${SSH_KEY_BASE} ]] ; then + die "SSH key directory ${SSH_KEY_BASE} not found" +fi + +BUBBLE_KEY_MARKER=/home/bubble/.refresh_ssh_keys +ROOT_KEY_MARKER=${SSH_KEY_BASE}/.refresh_ssh_keys + +if [[ ! -f ${BUBBLE_KEY_MARKER} ]] ; then + touch ${BUBBLE_KEY_MARKER} && chown bubble ${BUBBLE_KEY_MARKER} +fi +if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then + touch ${ROOT_KEY_MARKER} +fi + +log "Watching marker file ${BUBBLE_KEY_MARKER} ..." +while : ; do + if [[ $(stat -c %Y ${BUBBLE_KEY_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then + touch ${ROOT_KEY_MARKER} + sleep 5s + log "Refreshing Bubble SSH keys..." + /usr/local/sbin/refresh_bubble_ssh_keys.sh && log "Bubble SSH keys successfully refreshed" || log "Error refreshing Bubble SSH keys" + fi + sleep 10s +done diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_bubble_ssh_keys_monitor.conf b/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_bubble_ssh_keys_monitor.conf new file mode 100644 index 00000000..357fc6d8 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_bubble_ssh_keys_monitor.conf @@ -0,0 +1,5 @@ + +[program:refresh_bubble_ssh_keys_monitor] +stdout_logfile = /dev/null +stderr_logfile = /dev/null +command=/usr/local/sbin/refresh_bubble_ssh_keys_monitor.sh diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml new file mode 100644 index 00000000..55e8c2e6 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -0,0 +1,102 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install OpenJDK 11 JRE (headless), redis, uuid and jq + apt: + name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip' ] + state: present + update_cache: yes + +- import_tasks: postgresql.yml + +- name: Create bubble user + user: + name: bubble + comment: bubble user + shell: /bin/bash + system: yes + home: /home/bubble + +- name: Creates bubble API version dir + file: + path: /home/bubble/api + owner: bubble + group: bubble + mode: 0555 + state: directory + +- name: Creates bubble logs dir + file: + path: /home/bubble/logs + owner: bubble + group: root + mode: 0770 + state: directory + +- name: Install bubble jar + copy: + src: bubble.jar + dest: /home/bubble/api/bubble.jar + owner: bubble + group: bubble + mode: 0444 + +- name: Unpack site files + shell: cd /home/bubble && unzip -u /home/bubble/api/bubble.jar 'site/*' && chown -R bubble site + +- name: Install helper scripts + copy: + src: "{{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: root + group: root + mode: 0555 + with_items: + - "bsql.sh" + - "random_password.sh" + - "init_roles.sh" + +- name: Install standard bubble scripts + copy: + src: "{{ item }}" + dest: "/usr/local/bin/" + owner: root + group: root + mode: 0555 + with_fileglob: + - "scripts/*" + +- name: Link current version to the one we just installed + file: + src: /home/bubble/api + dest: /home/bubble/current + owner: bubble + group: bubble + state: link + +- name: Creates bubble SQL dir + file: + path: /home/bubble/sql + owner: bubble + group: postgres + mode: 0550 + state: directory + +- name: Install DB initializer + copy: + src: "{{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: root + group: postgres + mode: 0550 + with_items: + - init_bubble_db.sh + +- name: Install refresh_bubble_ssh_keys monitor + copy: + src: "refresh_bubble_ssh_keys_monitor.sh" + dest: "/usr/local/sbin/refresh_bubble_ssh_keys_monitor.sh" + owner: root + group: root + mode: 0500 + diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml new file mode 100644 index 00000000..780fef38 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml @@ -0,0 +1,36 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install PostgreSQL + apt: + name: [ 'postgresql-10', 'libpq-dev', 'python-psycopg2' ] + state: present + update_cache: yes + +- name: Install PostgreSQL files + copy: + src: "{{ item }}" + dest: /etc/postgresql/10/main/ + owner: postgres + group: postgres + mode: 0400 + with_items: + - postgresql.conf + - pg_hba.conf + +# When PostgreSQL restarts after a reboot, for some weird reason it looks in the 9.4 dir (instead of 10) for files +- name: Symlink /var/lib/postgresql/9.4 -> /var/lib/postgresql/10 + file: + src: /var/lib/postgresql/10 + dest: /var/lib/postgresql/9.4 + owner: root + group: root + state: link + +- name: Symlink /etc/postgresql/9.4 -> /etc/postgresql/10 + file: + src: /etc/postgresql/10 + dest: /etc/postgresql/9.4 + owner: root + group: root + state: link diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/bubble-nodemanager b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/bubble-nodemanager new file mode 100755 index 00000000..f236091f Binary files /dev/null and b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/bubble-nodemanager differ diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/copy_certs_to_bubble.sh b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/copy_certs_to_bubble.sh new file mode 100755 index 00000000..0a2d07e9 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/copy_certs_to_bubble.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +function die { + echo 1>&2 "${1}" + exit 1 +} + +MITM_CERTS=/home/mitmproxy/.mitmproxy +chown -R mitmproxy ${MITM_CERTS} || die "Error setting ownership on ${MITM_CERTS}" +chgrp -R root ${MITM_CERTS} || die "Error setting group on ${MITM_CERTS}" +chmod 750 ${MITM_CERTS} || die "Error setting permissions on ${MITM_CERTS}" +chmod -R 440 ${MITM_CERTS}/* || die "Error setting permissions on ${MITM_CERTS} files" + +CERTS_DIR=/home/bubble/cacerts + +CERT_BASE="${1:?no cert base provided}" +MITM_BASE_NAME="${CERT_BASE}-ca" + +mkdir -p ${CERTS_DIR} || die "Error creating cacerts dir" +cp ${MITM_CERTS}/${MITM_BASE_NAME}-cert.pem ${CERTS_DIR} || die "Error copying pem cert" +cp ${MITM_CERTS}/${MITM_BASE_NAME}-cert.pem.crt ${CERTS_DIR}/${MITM_BASE_NAME}-cert.crt || die "Error copying crt cert" +cp ${MITM_CERTS}/${MITM_BASE_NAME}-cert.p12 ${CERTS_DIR} || die "Error copying p12 cert" +cp ${MITM_CERTS}/${MITM_BASE_NAME}-cert.cer ${CERTS_DIR} || die "Error copying cer cert" +chown -R bubble ${CERTS_DIR} || die "Error setting permissions on cacerts dir" +chmod 755 ${CERTS_DIR} || die "Error setting permissions on ${CERTS_DIR}" +chmod -R 444 ${CERTS_DIR}/* || die "Error setting permissions on ${CERTS_DIR} files" + +CERTS_BACKUP=/home/bubble/mitm_certs +mkdir -p ${CERTS_BACKUP} || die "Error creating mitm_certs dir" +chmod 700 ${CERTS_BACKUP} || die "Error setting permissions on mitm_certs dir" +cp ${MITM_CERTS}/* ${CERTS_BACKUP} || die "Error backing up mitm_certs" +chmod -R 400 ${CERTS_BACKUP}/* || die "Error setting permissions on mitm_certs backup" +chown -R bubble ${CERTS_BACKUP} || die "Error settings ownership of mitm_certs dir" diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/supervisor_bubble_nodemanager.conf b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/supervisor_bubble_nodemanager.conf new file mode 100644 index 00000000..2605be5f --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/supervisor_bubble_nodemanager.conf @@ -0,0 +1,5 @@ + +[program:nodemanager] +stdout_logfile = /home/bubble/logs/nodemanager-out.log +stderr_logfile = /home/bubble/logs/nodemanager-err.log +command=/usr/sbin/bubble-nodemanager diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/tasks/main.yml new file mode 100644 index 00000000..ad8b2fcd --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble_finalizer/tasks/main.yml @@ -0,0 +1,36 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Touch first-time setup file + shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi" + when: restore_key is not defined + +- name: Install copy_certs_to_bubble.sh helper + copy: + src: "copy_certs_to_bubble.sh" + dest: /usr/local/bin/copy_certs_to_bubble.sh + owner: bubble + group: root + mode: 0550 + when: install_type == 'node' + +- name: Install bubble-nodemanager + copy: + src: "bubble-nodemanager" + dest: /usr/sbin/bubble-nodemanager + owner: root + group: root + mode: 0500 + +- name: Install bubble-nodemanager supervisor conf file + copy: + src: "supervisor_bubble_nodemanager.conf" + dest: /etc/supervisor/conf.d/nodemanager.conf + +# We cannot receive notifications until nginx is running, so start bubble API as the very last step +- name: Ensure bubble_nodemanager is started + supervisorctl: + name: '{{ item }}' + state: restarted + with_items: + - nodemanager diff --git a/bubble-server/src/main/resources/packer/roles/common/files/dot-screenrc b/bubble-server/src/main/resources/packer/roles/common/files/dot-screenrc new file mode 100644 index 00000000..fbe84fd1 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/common/files/dot-screenrc @@ -0,0 +1,24 @@ +bindkey -d ^J command +bind "\011" windows +escape ^Jj +defescape ^Jj +bind o focus +bind , prev +bind . next +bind k remove + +bind u focus down +bind i focus up +bind t focus top +bind b focus bottom + +vbell off + +# hardstatus on +# hardstatus alwayslastline + +defscrollback 50000 + +# Make it a login shell -- needed for rvm (Ruby) +shell -${SHELL} +term xterm-256color diff --git a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml new file mode 100644 index 00000000..4cd3ae90 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml @@ -0,0 +1,35 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Update packages + apt: + update_cache: yes + +- name: Remove ufw + apt: + name: ufw + state: absent + update_cache: yes + +- name: Upgrade packages + apt: + update_cache: yes + upgrade: yes + +- name: Install common packages + apt: + name: [ 'ntp', 'unzip', 'safe-rm', 'supervisor', 'emacs-nox', 'screen', 'xtail', 'fail2ban' ] + state: present + update_cache: yes + +- name: Install screenrc file + copy: + src: dot-screenrc + dest: /root/.screenrc + +- name: Start common services + service: + name: '{{ item }}' + state: restarted + with_items: + - fail2ban diff --git a/bubble-server/src/main/resources/packer/roles/firewall/defaults/main.yml b/bubble-server/src/main/resources/packer/roles/firewall/defaults/main.yml new file mode 100644 index 00000000..22915e36 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/defaults/main.yml @@ -0,0 +1,7 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +fw_enable_http: true +fw_enable_admin: true +fw_enable_dns: true +fw_enable_ssh: true diff --git a/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_peer_manager.py b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_peer_manager.py new file mode 100644 index 00000000..54732e3f --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_peer_manager.py @@ -0,0 +1,182 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +import json +import logging +import os +import sys +import time +import subprocess + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +EMPTY_PEERS = {'peers': [], 'ports': []} + + +class PeerPort(object): + def __init__(self, port): + if ':' in port: + self.proto = port[0:port.index(':')] + self.port = port[port.index(':') + 1:] + else: + self.proto = 'tcp' + self.port = port + + def __str__(self): + return self.proto + ':' + self.port + + +def find_peers(port): + out = subprocess.run(['iptables', '-vnL', 'INPUT'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + peers = [] + for line in out.stdout.decode('utf-8').split('\n'): + line = line.strip() + if len(line) == 0 or line.startswith('Chain ') or line.startswith('pkts '): + continue + for parts in line.split(' '): + packets = parts[0] + bytes = parts[1] + target = parts[2] + proto = parts[3] + if proto != port.proto: + continue + opt = parts[4] + iface_in = parts[5] + iface_out = parts[6] + source = parts[7] + if source == '0.0.0.0/0': + continue + dest = parts[8] + if parts[9] != port.proto: + continue + if parts[10].startswith('dpt:'): + dest_port = int(parts[10][len('dpt:'):]) + if dest_port == port.port: + peers.append(source) + return peers + + +def add_peers(peers, port): + out = subprocess.run(['iptables', '-vnL', 'INPUT'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + lines = out.stdout.decode('utf-8').split('\n') + insert_at = len(lines) - 2 + if insert_at < 2: + raise ValueError('add_peers: insert_at was < 2: '+str(insert_at)) + for peer in peers: + logger.info("add_peers: alllowing peer: " + peer + " on port " + port) + out = subprocess.run(['iptables', '-I', 'INPUT', str(insert_at), + '-p', port.proto, '-s', peer + '/32', + '--dport', port.port, '-j', 'ACCEPT']) + logger.info("add_peers: allowed peer: " + peer + " on port " + port) + + +def remove_peers(peers, port): + for peer in peers: + remove_peer(peer, port) + + +def remove_peer(peer, port): + out = subprocess.run(['iptables', '-vnL', 'INPUT'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + index = 0 + for line in out.stdout.decode('utf-8').split('\n'): + line = line.strip() + if len(line) == 0 or line.startswith('Chain ') or line.startswith('pkts '): + continue + index = index + 1 + for parts in line.split(' '): + packets = parts[0] + bytes = parts[1] + target = parts[2] + proto = parts[3] + if proto != port.proto: + continue + opt = parts[4] + iface_in = parts[5] + iface_out = parts[6] + source = parts[7] + if not source.startswith(peer+'/32'): + continue + dest = parts[8] + if parts[9] != port.proto: + continue + if parts[10].startswith('dpt:'): + dest_port = int(parts[10][len('dpt:'):]) + if dest_port == port.port: + logger.info("remove_peer: removing peer: " + peer + " on port " + port) + out = subprocess.run(['iptables', '-D', 'INPUT', str(index)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return True + return False + + +class BubblePeers(object): + + def __init__(self, peer_path, self_path): + self.peer_path = peer_path + if os.path.exists(peer_path): + self.last_modified = os.path.getmtime(self.peer_path) + else: + self.last_modified = 0 + + self.last_update = None + self.peers = [] + self.ports = [] + + self.self_path = self_path + self.self_node = {} + + def load_peers(self): + if os.path.exists(self.peer_path): + with open(self.peer_path) as f: + val = json.load(f) + else: + val = EMPTY_PEERS + self.peers = val['peers'] + self.ports = [] + for port in val['ports']: + self.ports.append(PeerPort(port)) + + def load_self(self): + if os.path.exists(self.self_path): + with open(self.self_path) as f: + self.self_node = json.load(f) + + def monitor(self): + self.load_peers() + self.load_self() + if os.path.exists(self.peer_path): + self.last_modified = os.path.getmtime(self.peer_path) + if self.last_update is None or self.last_update < self.last_modified: + self.load_peers() + for port in self.ports: + peers_on_port = find_peers(port) + peers_to_remove = [] + peers_to_add = [] + for peer in peers_on_port: + if peer not in self.peers: + peers_to_remove.append(peer) + for peer in self.peers: + if peer not in peers_on_port: + peers_to_add.append(peer) + remove_peers(peers_to_remove, port) + add_peers(peers_to_add, port) + + +if __name__ == "__main__": + peers = BubblePeers(sys.argv[1], sys.argv[2]) + interval = int(sys.argv[3]) + try: + while True: + peers.monitor() + time.sleep(interval) + except Exception as e: + logger.error("Unexpected error: " + repr(e)) diff --git a/bubble-server/src/main/resources/packer/roles/firewall/files/supervisor_bubble_peer_manager.conf b/bubble-server/src/main/resources/packer/roles/firewall/files/supervisor_bubble_peer_manager.conf new file mode 100644 index 00000000..01347d8d --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/files/supervisor_bubble_peer_manager.conf @@ -0,0 +1,5 @@ + +[program:bubble_peer_manager] +stdout_logfile = /var/log/bubble_peer_manager-out.log +stderr_logfile = /var/log/bubble_peer_manager-err.log +command=bash -c "/usr/local/bin/bubble_peer_manager.py /home/bubble/peers.json /home/bubble/self_node.json 60" diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml new file mode 100644 index 00000000..b0a94534 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml @@ -0,0 +1,121 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install firewall packages + apt: + name: [ 'haveged', 'iptables-persistent', 'netfilter-persistent' ] + state: present + update_cache: yes + +- name: Flush iptables + iptables: + flush: true + become: yes + +- name: Flush INPUT chain + iptables: + chain: INPUT + flush: yes + become: yes + +- name: Flush OUTPUT chain + iptables: + chain: OUTPUT + flush: yes + become: yes + +- name: Flush iptables nat table + iptables: + flush: yes + table: nat + become: yes + +- name: Flush iptables mangle table + iptables: + flush: true + table: mangle + become: yes + +- name: Flush iptables raw table + iptables: + flush: true + table: raw + become: yes + +- name: Flush OUTPUT chain NAT table + iptables: + chain: OUTPUT + table: nat + flush: yes + become: yes + +- name: Flush FORWARD chain + iptables: + chain: FORWARD + flush: yes + become: yes + +- name: Flush PREROUTING chain NAT Table + iptables: + chain: PREROUTING + table: nat + flush: yes + become: yes + +- name: Delete ufw chains + command: "bash -c 'iptables -F {{ item }} && iptables -X {{ item }} || echo \"chain not found: {{ item }}\"'" + with_items: + - ufw-after-forward + - ufw-after-input + - ufw-after-logging-forward + - ufw-after-logging-input + - ufw-after-logging-output + - ufw-after-output + - ufw-before-forward + - ufw-before-input + - ufw-before-logging-forward + - ufw-before-logging-input + - ufw-before-logging-output + - ufw-before-output + - ufw-reject-forward + - ufw-reject-input + - ufw-reject-output + - ufw-track-forward + - ufw-track-input + - ufw-track-output + +- name: Install port manager + copy: + src: bubble_peer_manager.py + dest: /usr/local/bin/bubble_peer_manager.py + owner: root + group: root + mode: 0555 + when: fw_enable_admin + +- name: Install supervisor conf file for port manager + copy: + src: supervisor_bubble_peer_manager.conf + dest: /etc/supervisor/conf.d/bubble_peer_manager.conf + when: fw_enable_admin + +- include: sage.yml + when: install_type == 'sage' + +- name: Creates /etc/iptables directory + file: + path: /etc/iptables + state: directory + +- name: save iptables v4 rules + shell: iptables-save > /etc/iptables/rules.v4 + become: yes + +- name: save iptables v6 rules + shell: ip6tables-save > /etc/iptables/rules.v6 + become: yes + +- supervisorctl: + name: bubble_peer_manager + state: restarted + when: fw_enable_admin diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/sage.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/sage.yml new file mode 100644 index 00000000..dcb24c85 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/sage.yml @@ -0,0 +1,61 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Allow all from local + iptables: + chain: INPUT + in_interface: lo + jump: ACCEPT + comment: Allow all from local + become: yes + +- name: Allow related and established connections + iptables: + chain: INPUT + ctstate: ESTABLISHED,RELATED + jump: ACCEPT + comment: Allow related and established connections + become: yes + +- name: Allow SSH + iptables: + chain: INPUT + protocol: tcp + destination_port: 22 + ctstate: NEW + syn: match + jump: ACCEPT + comment: Accept new SSH connections + become: yes + when: fw_enable_ssh + +- name: Allow HTTP + iptables: + chain: INPUT + protocol: tcp + destination_port: 80 + ctstate: NEW + syn: match + jump: ACCEPT + comment: Accept new HTTP connections + become: yes + when: fw_enable_http + +- name: Allow HTTPS + iptables: + chain: INPUT + protocol: tcp + destination_port: 443 + ctstate: NEW + syn: match + jump: ACCEPT + comment: Accept new HTTPS connections + become: yes + when: fw_enable_http + +- name: Drop everything else + iptables: + chain: INPUT + jump: DROP + comment: Drop anything else + become: yes diff --git a/bubble-server/src/main/resources/packer/roles/nginx/files/certbot_renew.sh b/bubble-server/src/main/resources/packer/roles/nginx/files/certbot_renew.sh new file mode 100644 index 00000000..bc184818 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/nginx/files/certbot_renew.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +if [[ -d /home/mitmproxy ]] ; then + service mitmproxy stop +fi +service nginx stop + +certbot renew --standalone --non-interactive || echo "Error updating SSL certificates" + +if [[ -d /home/mitmproxy ]] ; then + service mitmproxy restart +fi +service nginx restart diff --git a/bubble-server/src/main/resources/packer/roles/nginx/files/init_certbot.sh b/bubble-server/src/main/resources/packer/roles/nginx/files/init_certbot.sh new file mode 100755 index 00000000..6fca3964 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/nginx/files/init_certbot.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +LE_EMAIL="${1}" +SERVER_NAME="${2}" +SERVER_ALIAS="${3}" +if [[ $(find /etc/letsencrypt/accounts -type f -name regr.json | xargs grep -l \"${LE_EMAIL}\" | wc -l | tr -d ' ') -eq 0 ]] ; then + certbot register --agree-tos -m "${LE_EMAIL}" --non-interactive +fi + +if [[ ! -f /etc/letsencrypt/live/${SERVER_NAME}/fullchain.pem || ! -f /etc/letsencrypt/live/${SERVER_ALIAS}/fullchain.pem ]] ; then + certbot certonly --standalone --non-interactive -d ${SERVER_NAME} + certbot certonly --standalone --non-interactive -d ${SERVER_ALIAS} +else + certbot renew --standalone --non-interactive +fi diff --git a/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml new file mode 100644 index 00000000..2f10019c --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml @@ -0,0 +1,59 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install OpenSSL, nginx and software-properties-common + apt: + name: [ 'openssl', 'nginx', 'software-properties-common' ] + state: present + update_cache: yes + +- name: Enable Ubuntu universe repositories + apt_repository: + repo: "{{ item }}" + state: present + loop: + - "deb http://archive.ubuntu.com/ubuntu/ bionic universe" + - "deb http://archive.ubuntu.com/ubuntu/ bionic-updates universe" + - "deb http://security.ubuntu.com/ubuntu/ bionic-security universe" + +- name: Enable ppa:certbot/certbot repository + apt_repository: + repo: ppa:certbot/certbot + state: present + +- name: Update packages after adding new repositories + apt: + update_cache: yes + +- name: Install certbot + apt: + name: [ 'certbot' ] + state: present + update_cache: yes + +- name: Ensure nginx can read cert files + file: + dest: /etc/letsencrypt + group: www-data + recurse: yes + +- name: Ensure nginx is stopped + service: + name: nginx + state: stopped + +- name: Install init_certbot script + copy: + src: init_certbot.sh + dest: /usr/local/bin/init_certbot.sh + owner: root + group: root + mode: 0555 + +- name: Install certbot_renew.sh weekly cron job + copy: + src: "certbot_renew.sh" + dest: /etc/cron.weekly/certbot_renew.sh + owner: root + group: root + mode: 0755 diff --git a/bubble-server/src/test/resources/models/system/bubbleDomain.json b/bubble-server/src/test/resources/models/system/bubbleDomain.json index 3e594c5a..4803ac5c 100644 --- a/bubble-server/src/test/resources/models/system/bubbleDomain.json +++ b/bubble-server/src/test/resources/models/system/bubbleDomain.json @@ -3,15 +3,6 @@ "_subst": true, "name": "{{defaultDomain}}", "publicDns": "{{TEST_DEFAULT_DNS_CLOUD}}", - "template": true, - "roles": [ - "common-0.0.1", - "firewall-0.0.1", - "bubble-0.0.1", - "algo-0.0.1", - "mitmproxy-0.0.1", - "nginx-0.0.1", - "bubble_finalizer-0.0.1" - ] + "template": true } ] \ No newline at end of file diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index 45579ec8..5e49b6e3 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit 45579ec8407102a82dceae37df8d23f1ab1b9686 +Subproject commit 5e49b6e3e4281c59abd127279e90299f633db53f