diff --git a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java index f752f525e..a9bd919c1 100644 --- a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java +++ b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java @@ -34,6 +34,18 @@ */ public class MultiSemUtilities { + /** + * @return scan005 for w60_magc0399_scan005_m0013_r46_s01 + */ + public static String getScanStringForTileId(final String tileId) + throws IllegalArgumentException { + final int magcIndex = tileId.indexOf("magc"); + if ((magcIndex < 0) || (tileId.length() < (magcIndex + 16))) { + throw new IllegalArgumentException("scan string cannot be derived from tileId " + tileId); + } + return tileId.substring((magcIndex + 9), (magcIndex + 16)); // scan005; + } + /** * @return m0013 for w60_magc0399_scan005_m0013_r46_s01 */ diff --git a/render-app/src/test/java/org/janelia/alignment/multisem/MultiSemUtilitiesTest.java b/render-app/src/test/java/org/janelia/alignment/multisem/MultiSemUtilitiesTest.java index 4ee758f5a..86f0e54f1 100644 --- a/render-app/src/test/java/org/janelia/alignment/multisem/MultiSemUtilitiesTest.java +++ b/render-app/src/test/java/org/janelia/alignment/multisem/MultiSemUtilitiesTest.java @@ -21,6 +21,8 @@ public void testTileIdParsers() { "m0013_s01", MultiSemUtilities.getMfovSfovForTileId(tileId)); Assert.assertEquals("invalid SFOVIndexForTileId", "01", MultiSemUtilities.getSFOVIndexForTileId(tileId)); + Assert.assertEquals("invalid ScanStringForTileId", + "scan005", MultiSemUtilities.getScanStringForTileId(tileId)); final String manyScanTileId = "w66_magc0000_sc09876_m0005_r65_s16"; Assert.assertEquals("invalid MagcMfov", @@ -31,6 +33,8 @@ public void testTileIdParsers() { "m0005_s16", MultiSemUtilities.getMfovSfovForTileId(manyScanTileId)); Assert.assertEquals("invalid SFOVIndexForTileId", "16", MultiSemUtilities.getSFOVIndexForTileId(manyScanTileId)); + Assert.assertEquals("invalid ScanStringForTileId", + "sc09876", MultiSemUtilities.getScanStringForTileId(manyScanTileId)); } } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClient.java index fd02b105b..a049eadd8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RenderDataClient.java @@ -888,6 +888,34 @@ public Bounds getLayerBounds(final String stack, return httpClient.execute(httpGet, responseHandler); } + /** + * @param stack name of stack. + * @param z z value for layer. + * + * @return ids for all tiles in the specified layer of the specified stack. + * + * @throws IOException + * if the request fails for any reason. + */ + public List getTileIdsForZ(final String stack, + final Double z) + throws IOException { + + final URIBuilder builder = new URIBuilder(getUri(urls.getStackUrlString(stack) + "/tileIds")); + final URI uri = getUriWithZRangeParameters(z, z, builder); + + final HttpGet httpGet = new HttpGet(uri); + final String requestContext = "GET " + uri; + final TypeReference> typeReference = new TypeReference<>() { + }; + final JsonUtils.GenericHelper> helper = new JsonUtils.GenericHelper<>(typeReference); + final JsonResponseHandler> responseHandler = new JsonResponseHandler<>(requestContext, helper); + + LOG.info("getTileIdsForZ: submitting {}", requestContext); + + return httpClient.execute(httpGet, responseHandler); + } + /** * @param stack name of stack. * @param z z value for layer. diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MultiSEMTileRemovalClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MultiSEMTileRemovalClient.java new file mode 100644 index 000000000..32119719c --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MultiSEMTileRemovalClient.java @@ -0,0 +1,424 @@ +package org.janelia.render.client.multisem; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.janelia.alignment.multisem.MultiSemUtilities; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiSEMTileRemovalParameters; +import org.janelia.render.client.parameter.RenderWebServiceParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Java client that removes tiles from a multi-SEM stack. + *

+ * Removal is done in-place (the stack is set to the LOADING state, changed, and then completed) + * and supports the following operations: + *

    + *
  • removal of all tiles in one or more scans (see {@code --scan})
    • + *
  • removal of all tiles for one or more MFOVs in a specific scan (see {@code --scanMfov})
    • + *
  • renumbering of the layers that remain after layer removal (see {@code --collapseStack})
    • + *
+ * Scans are identified by name rather than by z value so that removal is idempotent. + * Note that stack bounds and other stats are recalculated when the stack is completed at the end of removal. + */ +public class MultiSEMTileRemovalClient { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); + + @Parameter( + names = "--stack", + description = "Stack from which tiles should be removed", + required = true) + public String stack; + + @ParametersDelegate + public MultiSEMTileRemovalParameters tileRemoval = new MultiSEMTileRemovalParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + LOG.info("runClient: entry, parameters={}", parameters); + + parameters.tileRemoval.validate(); + + final MultiSEMTileRemovalClient client = new MultiSEMTileRemovalClient(); + final RenderDataClient dataClient = parameters.renderWeb.getDataClient(); + client.removeTiles(dataClient, + parameters.stack, + parameters.tileRemoval); + } + }; + clientRunner.run(); + } + + public MultiSEMTileRemovalClient() { + } + + /** + * Removes the specified layers and MFOVs from the specified stack, + * collapsing the remaining layers if that was requested, + * and then completes the stack. + * + * @param dataClient client for the stack's owner and project. + * @param stack stack from which tiles should be removed. + * @param tileRemoval parameters identifying what should be removed. + * + * @throws IOException + * if any request fails. + */ + public void removeTiles(final RenderDataClient dataClient, + final String stack, + final MultiSEMTileRemovalParameters tileRemoval) + throws IOException { + + LOG.info("removeTiles: entry, stack={}, tileRemoval={}", stack, tileRemoval); + + final List stackZValues = getStackZValues(dataClient, stack); + + // map the requested scan names to this stack's z values so that removal is idempotent + final Map scanNameToZMap = buildScanNameToZMap(dataClient, stack, stackZValues, tileRemoval); + + // work out (and check) what should be removed before changing anything + final List zValuesToRemove = getZValuesToRemove(stack, tileRemoval, stackZValues, scanNameToZMap); + final List sortedRemainingZValues = stackZValues.stream() + .filter(z -> ! zValuesToRemove.contains(z)) + .sorted() + .collect(Collectors.toList()); + + final Map> zToMfovNamesMap = buildZToMfovNamesMap(tileRemoval, scanNameToZMap); + + validateMfovNames(dataClient, stack, zToMfovNamesMap, zValuesToRemove); + + // leave the stack alone if everything requested has already been removed + if (zValuesToRemove.isEmpty() && zToMfovNamesMap.isEmpty()) { + LOG.info("removeTiles: exit, nothing to remove from {}", stack); + return; + } + + dataClient.setStackState(stack, StackMetaData.StackState.LOADING); + + removeLayers(dataClient, stack, zValuesToRemove, sortedRemainingZValues.size()); + + if (! zToMfovNamesMap.isEmpty()) { + + int removedTileCount = 0; + for (final Double z : zToMfovNamesMap.keySet()) { + final Set mfovNames = zToMfovNamesMap.get(z); + if (sortedRemainingZValues.contains(z)) { + removedTileCount += removeMfovTilesForZ(dataClient, stack, z, mfovNames); + } else { + LOG.warn("removeTiles: skipping MFOVs {} because z {} is not in {}", mfovNames, z, stack); + } + } + + LOG.info("removeTiles: removed {} MFOV tiles from {} layers of {}", + removedTileCount, zToMfovNamesMap.size(), stack); + } + + if (tileRemoval.collapseStack) { + collapseStack(dataClient, stack, zValuesToRemove, sortedRemainingZValues); + } + + dataClient.setStackState(stack, StackMetaData.StackState.COMPLETE); + + LOG.info("removeTiles: exit, stack={}", stack); + } + + /** + * @return the sorted z values that exist in the stack before removal. + * + * @throws IOException + * if the stack does not exist or the request fails. + */ + private List getStackZValues(final RenderDataClient dataClient, + final String stack) + throws IOException { + + // fetch metadata first so that the run fails fast for stacks that do not exist + final StackMetaData stackMetaData = dataClient.getStackMetaData(stack); + + final List stackZValues = dataClient.getStackZValues(stack); + + LOG.info("getStackZValues: {} is in the {} state with {} layers", + stack, stackMetaData.getState(), stackZValues.size()); + + return stackZValues; + } + + /** + * @return map of z values to the simple names of the MFOVs that should be removed from each of those layers + * (scan names that are not in the specified map are excluded). + */ + private Map> buildZToMfovNamesMap(final MultiSEMTileRemovalParameters tileRemoval, + final Map scanNameToZMap) { + + final Map> zToMfovNamesMap = new TreeMap<>(); + + for (final Map.Entry> entry : tileRemoval.getScanToMfovNamesMap().entrySet()) { + final Double z = scanNameToZMap.get(entry.getKey()); + if (z != null) { + zToMfovNamesMap.computeIfAbsent(z, k -> new TreeSet<>()).addAll(entry.getValue()); + } + } + + return zToMfovNamesMap; + } + + /** + * Maps each scan name requested for removal to the z value of the layer that contains it. + * The scan name for a layer is derived from the first tile in that layer, + * so all tiles in a layer are assumed to come from the same scan. + * Requested scans that are no longer in the stack are simply logged + * (they have already been removed, so there is nothing left to do for them). + * + * @return map of scan names to z values for the layers that were checked. + * + * @throws IOException + * if a scan exists in more than one layer or if any request fails. + */ + private Map buildScanNameToZMap(final RenderDataClient dataClient, + final String stack, + final List stackZValues, + final MultiSEMTileRemovalParameters tileRemoval) + throws IOException { + + final Set requestedScanNames = tileRemoval.buildScanNamesSet(); + final Map scanNameToZMap = new HashMap<>(); + + for (final Double z : stackZValues) { + + final List tileIds = dataClient.getTileIdsForZ(stack, z); + + if (tileIds.isEmpty()) { + + LOG.warn("buildScanNameToZMap: no tiles exist for z {} in {}", z, stack); + + } else { + + final String scanName = MultiSemUtilities.getScanStringForTileId(tileIds.get(0)); + final Double previousZ = scanNameToZMap.put(scanName, z); + + if (previousZ != null) { + throw new IOException("scan " + scanName + " exists in z " + previousZ + " and z " + z + + " of " + stack); + } + + if (scanNameToZMap.keySet().containsAll(requestedScanNames)) { + break; // all requested scans have been found, so stop looking + } + } + } + + final List missingScanNames = requestedScanNames.stream() + .filter(scanName -> ! scanNameToZMap.containsKey(scanName)) + .collect(Collectors.toList()); + + if (! missingScanNames.isEmpty()) { + LOG.info("buildScanNameToZMap: nothing to remove for scan(s) {} because they no longer exist in {}", + missingScanNames, stack); + } + + LOG.info("buildScanNameToZMap: mapped {} of {} requested scan(s) to z values in {}", + (requestedScanNames.size() - missingScanNames.size()), requestedScanNames.size(), stack); + + return scanNameToZMap; + } + + /** + * @return the sorted z values for the layers that should be removed. + * + * @throws IllegalStateException + * if removing the layers would leave the stack empty. + */ + private List getZValuesToRemove(final String stack, + final MultiSEMTileRemovalParameters tileRemoval, + final List stackZValues, + final Map scanNameToZMap) + throws IllegalStateException { + + final List zValuesToRemove = tileRemoval.getSortedScanNames().stream() + .map(scanNameToZMap::get) + .filter(Objects::nonNull) + .distinct() + .sorted() + .collect(Collectors.toList()); + + // check before anything is changed so that a stack is not left without any layers + if (zValuesToRemove.size() == stackZValues.size()) { + throw new IllegalStateException("all " + stackZValues.size() + " layers would be removed from " + + stack + ", delete the stack instead if that is what you want"); + } + + return zValuesToRemove; + } + + /** + * Confirms that each MFOV requested for removal exists in its layer. + * MFOVs in layers that are being completely removed are logged and skipped. + * + * @throws IOException + * if a requested MFOV does not exist or if any request fails. + */ + private void validateMfovNames(final RenderDataClient dataClient, + final String stack, + final Map> zToMfovNamesMap, + final List zValuesToRemove) + throws IOException { + + for (final Map.Entry> entry : zToMfovNamesMap.entrySet()) { + + final Double z = entry.getKey(); + final Set mfovNames = entry.getValue(); + + if (zValuesToRemove.contains(z)) { + + LOG.warn("validateMfovNames: MFOVs {} do not need to be removed because all of z {} " + + "is being removed from {}", mfovNames, z, stack); + + } else { + + final Set layerMfovNames = dataClient.getTileIdsForZ(stack, z).stream() + .map(MultiSemUtilities::getSimpleMfovForTileId) + .collect(Collectors.toSet()); + + for (final String mfovName : mfovNames) { + if (! layerMfovNames.contains(mfovName)) { + throw new IOException("requested MFOV " + mfovName + " does not exist in z " + z + + " of " + stack); + } + } + } + } + } + + /** + * Removes all tiles in each of the specified layers. + * + * @throws IOException + * if any request fails. + */ + private void removeLayers(final RenderDataClient dataClient, + final String stack, + final List zValuesToRemove, + final int remainingLayerCount) + throws IOException { + + for (final Double z : zValuesToRemove) { + LOG.info("removeLayers: removing z {} from {}", z, stack); + dataClient.deleteStack(stack, z); + } + + LOG.info("removeLayers: removed {} layers from {} leaving {} layers", + zValuesToRemove.size(), stack, remainingLayerCount); + } + + /** + * Removes all tiles for the specified MFOVs from one layer. + * + * @return the number of tiles removed. + * + * @throws IOException + * if any request fails. + */ + private int removeMfovTilesForZ(final RenderDataClient dataClient, + final String stack, + final Double z, + final Set mfovNames) + throws IOException { + + final List tileIdsToRemove = dataClient.getTileIdsForZ(stack, z).stream() + .filter(tileId -> MultiSEMTileRemovalParameters.isTileInMfovs(tileId, mfovNames)) + .sorted() + .collect(Collectors.toList()); + + if (tileIdsToRemove.isEmpty()) { + LOG.warn("removeMfovTilesForZ: no tiles in z {} of {} are in MFOVs {}", + z, stack, mfovNames); + } else { + LOG.info("removeMfovTilesForZ: removing {} tiles from z {} of {} for MFOVs {}", + tileIdsToRemove.size(), z, stack, mfovNames); + for (final String tileId : tileIdsToRemove) { + dataClient.deleteStackTile(stack, tileId); + } + } + + return tileIdsToRemove.size(); + } + + /** + * Decreases the z value for each remaining layer by one for each removed layer before it. + *

+ * NOTE: layers are moved one at a time in ascending z order because each layer's tiles + * are identified by a query for its current z value. + * + * @throws IOException + * if any request fails. + */ + private void collapseStack(final RenderDataClient dataClient, + final String stack, + final List zValuesToRemove, + final List sortedRemainingZValues) + throws IOException { + + int movedLayerCount = 0; + + for (final Double z : sortedRemainingZValues) { + + final Double collapsedZ = getCollapsedZ(z, zValuesToRemove); + + if (! collapsedZ.equals(z)) { + + final List tileIds = dataClient.getTileIdsForZ(stack, z); + + LOG.info("collapseStack: moving {} tiles in {} from z {} to z {}", + tileIds.size(), stack, z, collapsedZ); + + if (! tileIds.isEmpty()) { + dataClient.updateZForTiles(stack, collapsedZ, tileIds); + movedLayerCount++; + } + } + } + + LOG.info("collapseStack: moved {} layers in {}", movedLayerCount, stack); + } + + /** + * @return the collapsed z value for the specified z + * (the specified z decreased by one for each removed layer before it). + */ + private Double getCollapsedZ(final Double z, + final List zValuesToRemove) { + int removedLayerCount = 0; + for (final Double removedZ : zValuesToRemove) { + if (removedZ < z) { + removedLayerCount++; + } + } + return z - removedLayerCount; + } + + private static final Logger LOG = LoggerFactory.getLogger(MultiSEMTileRemovalClient.class); +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiSEMTileRemovalParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiSEMTileRemovalParameters.java new file mode 100644 index 000000000..12903a0c1 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiSEMTileRemovalParameters.java @@ -0,0 +1,154 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.janelia.alignment.multisem.MultiSemUtilities; + +/** + * Parameters for removing tiles from one multi-SEM stack. + *

+ * Scans are identified by name instead of by z value so that removal is idempotent + * (scan names stay the same when layers are removed and renumbered). + */ +@Parameters +public class MultiSEMTileRemovalParameters + implements Serializable { + + @Parameter( + names = "--scan", + description = "scan name(s) for layer(s) that should be completely removed (e.g. scan004). " + + "Omit if no layers need to be removed.", + variableArity = true) + public List scanNames = new ArrayList<>(); + + @Parameter( + names = "--scanMfov", + description = "scan name and simple MFOV name for each MFOV that should be removed from one layer " + + "(e.g. scan004_m0015). Omit if no MFOVs need to be removed.", + variableArity = true) + public List scanMfovNames = new ArrayList<>(); + + @Parameter( + names = "--collapseStack", + description = "Indicates that the z value for each layer after a removed layer should be " + + "decreased by one for each removed layer before it " + + "(e.g. removing z 3 and 5 from a stack with z 1 through 6 leaves z 1, 2, 3, and 4)") + public boolean collapseStack = false; + + public MultiSEMTileRemovalParameters() { + } + + public boolean hasScanNames() { + return (scanNames != null) && (! scanNames.isEmpty()); + } + + public boolean hasScanMfovNames() { + return (scanMfovNames != null) && (! scanMfovNames.isEmpty()); + } + + public void validate() + throws IllegalArgumentException { + + if ((! hasScanNames()) && (! hasScanMfovNames())) { + throw new IllegalArgumentException("at least one --scan or --scanMfov value must be specified"); + } + + if (collapseStack && (! hasScanNames())) { + throw new IllegalArgumentException("--collapseStack requires at least one --scan value"); + } + + if (hasScanNames()) { + for (final String scanName : scanNames) { + if (! SCAN_NAME_PATTERN.matcher(scanName).matches()) { + throw new IllegalArgumentException( + "invalid --scan value '" + scanName + "', values must be scan names (e.g. scan004)"); + } + } + } + + // build the map to validate the format of all --scanMfov values + getScanToMfovNamesMap(); + } + + /** + * @return distinct sorted scan names for the layers that should be removed. + */ + public List getSortedScanNames() { + return hasScanNames() ? + scanNames.stream().distinct().sorted().collect(Collectors.toList()) : + new ArrayList<>(); + } + + /** + * @return map of scan names to the simple names of the MFOVs + * that should be removed from each of those layers. + * + * @throws IllegalArgumentException + * if any --scanMfov value is invalid. + */ + public Map> getScanToMfovNamesMap() + throws IllegalArgumentException { + + final Map> scanToMfovNamesMap = new TreeMap<>(); + + if (hasScanMfovNames()) { + for (final String scanMfovName : scanMfovNames) { + + final Matcher matcher = SCAN_MFOV_NAME_PATTERN.matcher(scanMfovName); + if (! matcher.matches()) { + throw new IllegalArgumentException( + "invalid --scanMfov value '" + scanMfovName + "', values must specify a scan name " + + "and a simple MFOV name (e.g. scan004_m0015)"); + } + + scanToMfovNamesMap.computeIfAbsent(matcher.group(1), k -> new TreeSet<>()).add(matcher.group(2)); + } + } + + return scanToMfovNamesMap; + } + + /** + * @return all distinct scan names referenced by the --scan and --scanMfov values. + */ + public Set buildScanNamesSet() { + final Set allScanNames = new TreeSet<>(getSortedScanNames()); + allScanNames.addAll(getScanToMfovNamesMap().keySet()); + return allScanNames; + } + + /** + * @return true if the specified tile is in one of the specified MFOVs, otherwise false. + */ + public static boolean isTileInMfovs(final String tileId, + final Set mfovNames) + throws IllegalArgumentException { + return mfovNames.contains(MultiSemUtilities.getSimpleMfovForTileId(tileId)); + } + + @Override + public String toString() { + return "{scanNames=" + scanNames + + ", scanMfovNames=" + scanMfovNames + + ", collapseStack=" + collapseStack + + '}'; + } + + /** Matches --scan values like scan004 (and sc01234). */ + private static final Pattern SCAN_NAME_PATTERN = Pattern.compile("^sc[^_]+$"); + + /** Matches --scanMfov values like scan004_m0015 (and sc01234_m0015). */ + private static final Pattern SCAN_MFOV_NAME_PATTERN = Pattern.compile("^(sc[^_]+)_(m\\d{4})$"); +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackWithRemovalParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackWithRemovalParameters.java new file mode 100644 index 000000000..bcad4ce12 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackWithRemovalParameters.java @@ -0,0 +1,61 @@ +package org.janelia.render.client.parameter; + +import java.io.Serializable; + +import org.janelia.alignment.spec.stack.StackId; + +/** + * Couples a stack with the parameters for removing tiles from that stack. + */ +public class StackWithRemovalParameters + implements Serializable { + + private final StackId stackId; + private final MultiSEMTileRemovalParameters tileRemoval; + + // no-arg constructor needed for JSON deserialization + @SuppressWarnings("unused") + public StackWithRemovalParameters() { + this(null, null); + } + + public StackWithRemovalParameters(final StackId stackId, + final MultiSEMTileRemovalParameters tileRemoval) { + this.stackId = stackId; + this.tileRemoval = tileRemoval; + } + + public StackId getStackId() { + return stackId; + } + + public MultiSEMTileRemovalParameters getTileRemoval() { + return tileRemoval; + } + + public void validate() + throws IllegalArgumentException { + + if (stackId == null) { + throw new IllegalArgumentException("stackId must be defined for each tile removal element"); + } + + if (tileRemoval == null) { + throw new IllegalArgumentException("tileRemoval must be defined for " + stackId.toDevString()); + } + + try { + tileRemoval.validate(); + } catch (final IllegalArgumentException e) { + throw new IllegalArgumentException("invalid tile removal parameters for " + stackId.toDevString() + + ": " + e.getMessage(), e); + } + } + + @Override + public String toString() { + return "{stackId=" + (stackId == null ? null : stackId.toDevString()) + + ", tileRemoval=" + tileRemoval + + '}'; + } +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MultiSEMTileRemovalClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MultiSEMTileRemovalClient.java new file mode 100644 index 000000000..cb1300c33 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MultiSEMTileRemovalClient.java @@ -0,0 +1,152 @@ +package org.janelia.render.client.spark.multisem; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.parameter.MultiSEMTileRemovalParameters; +import org.janelia.render.client.parameter.StackWithRemovalParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for removing tiles from multi-SEM stacks. + * Core logic is implemented in {@link org.janelia.render.client.multisem.MultiSEMTileRemovalClient}. + * + *

Removal operations are quick web service calls, so nothing is distributed to spark workers here.

+ * + * @see org.janelia.render.client.multisem.MultiSEMTileRemovalClient + * + * @author Eric Trautman + */ +public class MultiSEMTileRemovalClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public MultiSEMTileRemovalParameters tileRemoval = new MultiSEMTileRemovalParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + parameters.tileRemoval.validate(); + + LOG.info("runClient: entry, parameters={}", parameters); + + // apply the same removal parameters to each stack identified by the multiProject parameters + final MultiProjectParameters multiProject = parameters.multiProject; + final List stackWithRemovalList = + multiProject.stackIdWithZ.getStackIdList(multiProject.getDataClient()).stream() + .map(stackId -> new StackWithRemovalParameters(stackId, parameters.tileRemoval)) + .collect(Collectors.toList()); + + // NOTE: no spark context is needed here because all removal is run on the driver + final MultiSEMTileRemovalClient client = new MultiSEMTileRemovalClient(); + client.removeTiles(multiProject.getBaseDataUrl(), stackWithRemovalList); + } + }; + clientRunner.run(); + } + + /** Empty constructor required for alignment pipeline steps. */ + public MultiSEMTileRemovalClient() { + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + + final List stackWithRemovalList = + pipelineParameters.getTileRemovalList(); + + AlignmentPipelineParameters.validateRequiredElementExists("tileRemovalList", + stackWithRemovalList); + + if (stackWithRemovalList.isEmpty()) { + throw new IllegalArgumentException("tileRemovalList must contain at least one element"); + } + + for (final StackWithRemovalParameters stackWithRemoval : stackWithRemovalList) { + stackWithRemoval.validate(); + } + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + removeTiles(pipelineParameters.getMultiProject(null).getBaseDataUrl(), + pipelineParameters.getTileRemovalList()); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.REMOVE_TILES; + } + + public void removeTiles(final String baseDataUrl, + final List stackWithRemovalList) + throws IOException { + + LOG.info("removeTiles: entry, processing {} stack(s)", stackWithRemovalList.size()); + + final org.janelia.render.client.multisem.MultiSEMTileRemovalClient javaClient = + new org.janelia.render.client.multisem.MultiSEMTileRemovalClient(); + + // cache each owner's stack ids so that existence can be checked without repeating requests + final Map> ownerToStackIds = new HashMap<>(); + + for (final StackWithRemovalParameters stackWithRemoval : stackWithRemovalList) { + + final StackId stackId = stackWithRemoval.getStackId(); + final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, + stackId.getOwner(), + stackId.getProject()); + + Set ownerStackIds = ownerToStackIds.get(stackId.getOwner()); + if (ownerStackIds == null) { + ownerStackIds = new HashSet<>(dataClient.getOwnerStacks()); + ownerToStackIds.put(stackId.getOwner(), ownerStackIds); + } + + if (ownerStackIds.contains(stackId)) { + LOG.info("removeTiles: processing {}", stackId.toDevString()); + javaClient.removeTiles(dataClient, stackId.getStack(), stackWithRemoval.getTileRemoval()); + } else { + LOG.info("removeTiles: skipping removal for {} because the stack does not exist", + stackId.toDevString()); + } + } + + LOG.info("removeTiles: exit"); + } + + private static final Logger LOG = LoggerFactory.getLogger(MultiSEMTileRemovalClient.class); +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index c899250af..78cf3ac5e 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -26,6 +26,7 @@ import org.janelia.render.client.parameter.MatchCopyParameters; import org.janelia.render.client.parameter.MipmapParameters; import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.parameter.StackWithRemovalParameters; import org.janelia.render.client.parameter.ScapeParameters; import org.janelia.render.client.parameter.TileClusterParameters; import org.janelia.render.client.parameter.TileRenderParameters; @@ -63,6 +64,7 @@ public class AlignmentPipelineParameters private final TileRenderParameters tileRender; private final MFOVAsTileParameters mfovAsTile; private final LayerAsTileParameters layerAsTile; + private final List tileRemovalList; @SuppressWarnings("unused") public AlignmentPipelineParameters() { @@ -85,6 +87,7 @@ public AlignmentPipelineParameters() { null, null, null, + null, null); } @@ -107,7 +110,8 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, final ScapeParameters scape, final TileRenderParameters tileRender, final MFOVAsTileParameters mfovAsTile, - final LayerAsTileParameters layerAsTile) { + final LayerAsTileParameters layerAsTile, + final List tileRemovalList) { this.multiProject = multiProject; this.pipelineStackGroups = pipelineStackGroups; this.pipelineSteps = pipelineSteps; @@ -128,6 +132,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, this.tileRender = tileRender; this.mfovAsTile = mfovAsTile; this.layerAsTile = layerAsTile; + this.tileRemovalList = tileRemovalList; } public MultiProjectParameters getMultiProject(final StackIdNamingGroup withNamingGroup) { @@ -228,6 +233,10 @@ public LayerAsTileParameters getLayerAsTile() { return layerAsTile; } + public List getTileRemovalList() { + return tileRemovalList; + } + /** * @return a list of clients for each specified pipeline step. * diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index 9dba4fb2e..e2c1b8234 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -14,6 +14,7 @@ import org.janelia.render.client.spark.multisem.MFOVAsTileClient; import org.janelia.render.client.spark.multisem.MFOVMontageMatchPatchClient; import org.janelia.render.client.spark.multisem.MatchCollectionRenameClient; +import org.janelia.render.client.spark.multisem.MultiSEMTileRemovalClient; import org.janelia.render.client.spark.multisem.UnconnectedCrossMFOVClient; import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; import org.janelia.render.client.spark.newsolver.DistributedIntensityCorrectionBlockSolverClient; @@ -39,6 +40,7 @@ public enum AlignmentPipelineStepId { CORRECT_INTENSITY(DistributedIntensityCorrectionBlockSolverClient::new), HACK_MASK(MaskHackClient::new), HACK_TILE_ID(TileIdHackClient::new), + REMOVE_TILES(MultiSEMTileRemovalClient::new), RENDER_SCAPE_IMAGES(ScapeClient::new), RENDER_TILES(RenderTilesClient::new), MFOV_AS_TILE(MFOVAsTileClient::new), diff --git a/render-ws-with-mongo-db/other/load-rough-align-data.sh b/render-ws-with-mongo-db/other/load-rough-align-data.sh deleted file mode 100644 index f4067d8d1..000000000 --- a/render-ws-with-mongo-db/other/load-rough-align-data.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -# Batch identifier appended to each slab group name (edit this for each round of runs). -SLAB_GROUP_SUFFIX="20260811b" - -VM_IPS=(10.150.0.2 10.150.0.3 10.150.0.4 10.150.0.5 10.150.0.6 10.150.0.7) - -printf "\nWhich VM do you want to use?\n\n" -select VM_IP in "${VM_IPS[@]}"; do - if [ -n "${VM_IP}" ]; then - break - else - echo "Invalid selection, try again." - fi -done - -printf "\nWhich wafer do you want to use?\n\n" -select WAFER in 60 61; do - if [ -n "${WAFER}" ]; then - break - else - echo "Invalid selection, try again." - fi -done - -echo -read -rp "Enter the first serial number (a multiple of 5 between 0 and 410): " FIRST_SERIAL_NUMBER - -if [[ ! ${FIRST_SERIAL_NUMBER} =~ ^[0-9]+$ ]]; then - printf "\nExiting, '%s' is not a number\n\n" "${FIRST_SERIAL_NUMBER}" - exit 1 -fi - -# force base 10 so that zero padded values (e.g. 070) are not treated as octal -FIRST_SERIAL_NUMBER=$(( 10#${FIRST_SERIAL_NUMBER} )) - -if (( FIRST_SERIAL_NUMBER > 410 )) || (( FIRST_SERIAL_NUMBER % 5 != 0 )); then - printf "\nExiting, %d is not a multiple of 5 between 0 and 410\n\n" "${FIRST_SERIAL_NUMBER}" - exit 1 -fi - -LAST_SERIAL_NUMBER=$(( FIRST_SERIAL_NUMBER + 4 )) - -# even serial numbers are the first half of a project's slabs, odd ones are the second half -if (( FIRST_SERIAL_NUMBER % 2 == 0 )); then - KEEP_OR_REMOVE="[k]ept" - FIRST_PROJECT_NUMBER=${FIRST_SERIAL_NUMBER} -else - KEEP_OR_REMOVE="[r]emoved" - FIRST_PROJECT_NUMBER=$(( FIRST_SERIAL_NUMBER - 5 )) -fi - -LAST_PROJECT_NUMBER=$(( FIRST_PROJECT_NUMBER + 9 )) - -FIRST_SERIAL=$(printf "%03d" "${FIRST_SERIAL_NUMBER}") -LAST_SERIAL=$(printf "%03d" "${LAST_SERIAL_NUMBER}") -FIRST_PROJECT=$(printf "%03d" "${FIRST_PROJECT_NUMBER}") -LAST_PROJECT=$(printf "%03d" "${LAST_PROJECT_NUMBER}") - -SLAB_GROUP="s${FIRST_SERIAL}_to_s${LAST_SERIAL}_${SLAB_GROUP_SUFFIX}" -BATCH_NAME="rough-w${WAFER}-s${FIRST_SERIAL}-to-s${LAST_SERIAL}" -PROJECT_GROUP="w${WAFER}_serial_${FIRST_PROJECT}_to_${LAST_PROJECT}" - -echo " -Set up for slab group ${SLAB_GROUP} from project group ${PROJECT_GROUP}: - - On ${VM_IP}, run: - - ./db-restore-collections.sh --pattern 'janelia/00_gc/.*s${FIRST_PROJECT}' - - ./other/remove-stacks.sh - - you want stacks to be ${KEEP_OR_REMOVE} - then enter ' 1 2 3 4 5 6 7 8 9 10 ' - - - On launch box, run: - - ./02_run_pipeline.sh ${VM_IP} 00_rough_align/pipe.00.w${WAFER}.icc-match-mat.json 120 4 premium 120 ${BATCH_NAME} - - - After the run completes (typically 8 to 12 hours), on ${VM_IP}, run: - - ./list-stacks.sh - - ./list-match-collections.sh - - ./db-dump-google-collections.sh - - Select database: render - Select stage: 00_par - Select project: ${PROJECT_GROUP} - Enter slab-group: ${SLAB_GROUP} - - Enter collection pattern regex: .* - - Should dump collections to: - /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/render - - ./db-dump-google-collections.sh - - Select database: match - Select stage: 00_par - Select project: ${PROJECT_GROUP} - Enter slab-group: ${SLAB_GROUP} - - Enter collection pattern regex: .* - - Should dump collections to: - /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/match -"