diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index 15879fb47649..05ddf4b9e4f9 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -213,6 +213,35 @@ public static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient, return getBlock(xceiverClient, getValidatorList(), datanodeBlockID, token, replicaIndexes); } + /** + * Gets block metadata from a datanode. + *

+ * + * @param xceiverClient client to perform call + * @param blockID blockID to identify container + * @param token a token for this block (may be null) + * @param datanode datanode to query + * @param replicaIndexes replica indexes for EC pipelines + * @return container protocol get block response + * @throws IOException if there is an I/O error while performing the call + */ + public static GetBlockResponseProto getBlockFromDatanode( + XceiverClientSpi xceiverClient, + BlockID blockID, + Token token, + DatanodeDetails datanode, + Map replicaIndexes) throws IOException { + ContainerCommandRequestProto.Builder builder = ContainerCommandRequestProto + .newBuilder() + .setCmdType(Type.GetBlock) + .setContainerID(blockID.getContainerID()); + if (token != null) { + builder.setEncodedToken(token.encodeToUrlString()); + } + return getBlock(xceiverClient, getValidatorList(), builder, blockID, datanode, + replicaIndexes); + } + private static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient, List validators, ContainerCommandRequestProto.Builder builder, BlockID blockID, diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/chunk/ChunkKeyHandler.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/chunk/ChunkKeyHandler.java index 4f53d02f2339..146354119220 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/chunk/ChunkKeyHandler.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/chunk/ChunkKeyHandler.java @@ -126,11 +126,11 @@ protected void execute(OzoneClient client, OzoneAddress address) // Process each datanode individually for (DatanodeDetails datanodeDetails : pipeline.getNodes()) { try { - // Get block from THIS ONE datanode only ContainerProtos.GetBlockResponseProto blockResponse = - ContainerProtocolCalls.getBlock(xceiverClient, + ContainerProtocolCalls.getBlockFromDatanode(xceiverClient, keyLocation.getBlockID(), keyLocation.getToken(), + datanodeDetails, pipeline.getReplicaIndexes()); if (blockResponse == null || !blockResponse.hasBlockData()) { diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh index 6363e9cde5b7..2e3552fc969b 100755 --- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh @@ -30,4 +30,5 @@ execute_robot_test scm -v PREFIX:${prefix} -N read-3-datanodes ec/read.robot docker-compose up -d --no-recreate --scale datanode=5 execute_robot_test scm -v container:1 -v count:5 -N EC-recovery replication/wait.robot docker-compose up -d --no-recreate --scale datanode=9 +execute_robot_test scm -v PREFIX:${prefix} -N debug-ec6-3 debug/ozone-debug-tests-ec6-3.robot execute_robot_test scm -N S3-EC-Storage ec/awss3ecstorage.robot diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config index 8133eb1073e6..5fb05c5a10dc 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config @@ -42,6 +42,7 @@ OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s OZONE-SITE.XML_ozone.om.volume.listall.allowed=false OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.scm.block.size=1MB OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index e204cc70d756..c8ef489a600a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -24,6 +24,7 @@ OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.http-address=scm:9876 OZONE-SITE.XML_ozone.scm.container.size=1GB +OZONE-SITE.XML_ozone.scm.block.size=1MB OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1 diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot index aa51febb318e..018aa976ce8c 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot @@ -101,3 +101,23 @@ Get key names from output Append To List ${key_names} ${key_name} END [Return] ${key_names} + +Get chunk-info block sizes by group + ${output} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} | jq -c '[.keyLocations[] | [.[] | {i: .replicaIndex, s: .blockData.size}] | sort_by(.i) | map(.s)]' + [Return] ${output} + +Verify chunk-info block sizes + [Arguments] ${expected_json} + ${actual_json} = Get chunk-info block sizes by group + ${actual} = Evaluate json.dumps(json.loads('''${actual_json}'''.strip())) json, json + ${expected} = Evaluate json.dumps(json.loads('''${expected_json}''')) json, json + Should Be Equal As Strings ${actual} ${expected} + +Create EC key + [Arguments] ${ec_data} ${ec_parity} ${file_size} + Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=1 count=${file_size} + Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${ec_data}-${ec_parity}-1024k -t EC + +Create Volume Bucket + Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} + Execute ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot index 7b88f97254c9..fb2b91c47a51 100644 --- a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot @@ -30,20 +30,32 @@ ${TESTFILE} testfile ${EC_DATA} 3 ${EC_PARITY} 2 ${OM_SERVICE_ID} %{OM_SERVICE_ID} +# single-block stripe: one full data block (1048576), other data blocks=0, parity blocks mirrors data1 +${EC32_SINGLE_BLOCK_STRIPE_SIZES} [[1048576,0,0,1048576,1048576]] +# 3 MiB full stripe (3*1048576): all 5 replicas are one full 1024k chunk +${EC32_FULL_STRIPE_SIZES} [[1048576,1048576,1048576,1048576,1048576]] +# group0: full stripe; group1: partial-block stripe with one 1000000 B data block +${EC32_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES} [[1048576,1048576,1048576,1048576,1048576],[1000000,0,0,1000000,1000000]] +# multi-block stripe: data1-2=1048576, data3=2500000%1048576=402848, parity mirrors data1 +${EC32_MULTI_BLOCK_STRIPE_SIZES} [[1048576,1048576,402848,1048576,1048576]] -*** Keywords *** -Create Volume Bucket - Execute ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME} - Execute ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET} +*** Test Cases *** +Test ozone debug replicas chunk-info single-block stripe + # 1*1048576: one full data block in a single-block stripe + Create EC key ${EC_DATA} ${EC_PARITY} 1048576 + Verify chunk-info block sizes ${EC32_SINGLE_BLOCK_STRIPE_SIZES} -Create EC key - [arguments] ${bs} ${count} +Test ozone debug replicas chunk-info full stripe + # 3*1048576: EC_DATA full 1024k chunks = one complete stripe + Create EC key ${EC_DATA} ${EC_PARITY} 3145728 + Verify chunk-info block sizes ${EC32_FULL_STRIPE_SIZES} - Execute dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count} - Execute ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC +Test ozone debug replicas chunk-info full stripe and partial-block stripe + # 3*1048576 + 1000000: one full stripe plus a partial-block stripe (1000000 B data block) + Create EC key ${EC_DATA} ${EC_PARITY} 4145728 + Verify chunk-info block sizes ${EC32_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES} -*** Test Cases *** -Test ozone debug replicas chunk-info - Create EC key 1048576 3 - ${count} = Execute ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length' - Should Be Equal As Integers ${count} 5 +Test ozone debug replicas chunk-info multi-block stripe + # 2*1048576 + 402848: two full data blocks plus a 402848 B partial data block + Create EC key ${EC_DATA} ${EC_PARITY} 2500000 + Verify chunk-info block sizes ${EC32_MULTI_BLOCK_STRIPE_SIZES} diff --git a/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot new file mode 100644 index 000000000000..0459f468c505 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test ozone Debug CLI for EC(6,3) replicated keys +Library OperatingSystem +Library Process +Resource ../ec/lib.resource +Resource ../lib/os.robot +Resource ozone-debug-keywords.robot +Test Timeout 5 minute +Suite Setup Run Keywords Wait Until Keyword Succeeds 2min 10sec Has Enough Datanodes 9 +... AND Create Volume Bucket + +*** Variables *** +${PREFIX} ${EMPTY} +${VOLUME} cli-debug-ec6-volume${PREFIX} +${BUCKET} cli-debug-ec6-bucket +${TESTFILE} testfile +${EC_DATA} 6 +${EC_PARITY} 3 +${OM_SERVICE_ID} %{OM_SERVICE_ID} +# single-block stripe: one full data block (1048576), other data blocks=0, parity mirrors data1 +${EC63_SINGLE_BLOCK_STRIPE_SIZES} [[1048576,0,0,0,0,0,1048576,1048576,1048576]] +# 6 MiB full stripe (6*1048576): all 9 replicas are one full 1024k chunk +${EC63_FULL_STRIPE_SIZES} [[1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576]] +# group0: full stripe; group1: partial-block stripe with one 1000000 B data block +${EC63_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES} [[1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576],[1000000,0,0,0,0,0,1000000,1000000,1000000]] +# multi-block stripe: data1-3=1048576, data4=3500000%1048576=354272, data5-6=0, parity mirrors data1 +${EC63_MULTI_BLOCK_STRIPE_SIZES} [[1048576,1048576,1048576,354272,0,0,1048576,1048576,1048576]] + +*** Test Cases *** +Test ozone debug replicas chunk-info single-block stripe + # 1*1048576: one full data block in a single-block stripe + Create EC key ${EC_DATA} ${EC_PARITY} 1048576 + Verify chunk-info block sizes ${EC63_SINGLE_BLOCK_STRIPE_SIZES} + +Test ozone debug replicas chunk-info full stripe + # 6*1048576: EC_DATA full 1024k chunks = one complete stripe + Create EC key ${EC_DATA} ${EC_PARITY} 6291456 + Verify chunk-info block sizes ${EC63_FULL_STRIPE_SIZES} + +Test ozone debug replicas chunk-info full stripe and partial-block stripe + # 6*1048576 + 1000000: one full stripe plus a partial-block stripe (1000000 B data block) + Create EC key ${EC_DATA} ${EC_PARITY} 7291456 + Verify chunk-info block sizes ${EC63_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES} + +Test ozone debug replicas chunk-info multi-block stripe + # 3*1048576 + 354272: three full data blocks plus a 354272 B partial data block + Create EC key ${EC_DATA} ${EC_PARITY} 3500000 + Verify chunk-info block sizes ${EC63_MULTI_BLOCK_STRIPE_SIZES} diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/awss3ecstorage.robot b/hadoop-ozone/dist/src/main/smoketest/ec/awss3ecstorage.robot index 07908107ea85..de3387bb3d3a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/awss3ecstorage.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ec/awss3ecstorage.robot @@ -18,6 +18,7 @@ Documentation S3 gateway test with aws cli with STANDARD_IA storage class Library OperatingSystem Library String Resource ../commonlib.robot +Resource lib.resource Resource ../s3/commonawslib.robot Resource ../s3/mpu_lib.robot Resource ../ozone-lib/shell.robot @@ -34,16 +35,6 @@ Setup EC Multipart Tests Teardown EC Multipart Tests Remove Files /tmp/1mb -Count Datanodes In Service - ${actual} = Execute ozone admin datanode list --node-state HEALTHY --operational-state IN_SERVICE --json | jq -r 'length' - [return] ${actual} - -Has Enough Datanodes - [arguments] ${expected} - ${actual} = Count Datanodes In Service - Should Be True ${expected} <= ${actual} - - *** Variables *** ${ENDPOINT_URL} http://s3g:9878 ${BUCKET} generated diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource b/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource index 63b7250e205e..3ddd87a12d24 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource +++ b/hadoop-ozone/dist/src/main/smoketest/ec/lib.resource @@ -24,6 +24,15 @@ Suite Setup Get Security Enabled From Config ${SCM} scm *** Keywords *** +Count Datanodes In Service + ${actual} = Execute ozone admin datanode list --node-state HEALTHY --operational-state IN_SERVICE --json | jq -r 'length' + [return] ${actual} + +Has Enough Datanodes + [arguments] ${expected} + ${actual} = Count Datanodes In Service + Should Be True ${expected} <= ${actual} + Prepare For Tests Execute dd if=/dev/urandom of=/tmp/1mb bs=1048576 count=1 Execute dd if=/dev/urandom of=/tmp/2mb bs=1048576 count=2