HDDS-15581. ozone debug replicas chunk-info reports wrong block size for EC keys

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java

@@ -213,6 +213,35 @@ public static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient,
     return getBlock(xceiverClient, getValidatorList(), datanodeBlockID, token, replicaIndexes);
   }
 
+  /**
+   * Gets block metadata from a datanode.
+   * <p>
+   *
+   * @param xceiverClient client to perform call
+   * @param blockID blockID to identify container
+   * @param token a token for this block (may be null)
+   * @param datanode datanode to query
+   * @param replicaIndexes replica indexes for EC pipelines
+   * @return container protocol get block response
+   * @throws IOException if there is an I/O error while performing the call
+   */
+  public static GetBlockResponseProto getBlockFromDatanode(
+      XceiverClientSpi xceiverClient,
+      BlockID blockID,
+      Token<? extends TokenIdentifier> token,
+      DatanodeDetails datanode,
+      Map<DatanodeDetails, Integer> replicaIndexes) throws IOException {
+    ContainerCommandRequestProto.Builder builder = ContainerCommandRequestProto
+        .newBuilder()
+        .setCmdType(Type.GetBlock)
+        .setContainerID(blockID.getContainerID());
+    if (token != null) {
+      builder.setEncodedToken(token.encodeToUrlString());
+    }
+    return getBlock(xceiverClient, getValidatorList(), builder, blockID, datanode,
+        replicaIndexes);
+  }
+
   private static GetBlockResponseProto getBlock(XceiverClientSpi xceiverClient,
       List<Validator> validators,
       ContainerCommandRequestProto.Builder builder, BlockID blockID,

hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/chunk/ChunkKeyHandler.java

@@ -126,11 +126,11 @@ protected void execute(OzoneClient client, OzoneAddress address)
             // Process each datanode individually
             for (DatanodeDetails datanodeDetails : pipeline.getNodes()) {
               try {
-                // Get block from THIS ONE datanode only
                 ContainerProtos.GetBlockResponseProto blockResponse =
-                    ContainerProtocolCalls.getBlock(xceiverClient,
+                    ContainerProtocolCalls.getBlockFromDatanode(xceiverClient,
                         keyLocation.getBlockID(),
                         keyLocation.getToken(),
+                        datanodeDetails,
                         pipeline.getReplicaIndexes());
 
                 if (blockResponse == null || !blockResponse.hasBlockData()) {

hadoop-ozone/dist/src/main/compose/common/ec-test.sh

@@ -30,4 +30,5 @@ execute_robot_test scm -v PREFIX:${prefix} -N read-3-datanodes ec/read.robot
 docker-compose up -d --no-recreate --scale datanode=5
 execute_robot_test scm -v container:1 -v count:5 -N EC-recovery replication/wait.robot
 docker-compose up -d --no-recreate --scale datanode=9
+execute_robot_test scm -v PREFIX:${prefix} -N debug-ec6-3 debug/ozone-debug-tests-ec6-3.robot
 execute_robot_test scm -N S3-EC-Storage ec/awss3ecstorage.robot

hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config

@@ -42,6 +42,7 @@ OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s
 OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
 
 OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.block.size=1MB
 OZONE-SITE.XML_ozone.scm.datanode.ratis.volume.free-space.min=10MB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -24,6 +24,7 @@ OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.http-address=scm:9876
 OZONE-SITE.XML_ozone.scm.container.size=1GB
+OZONE-SITE.XML_ozone.scm.block.size=1MB
 OZONE-SITE.XML_ozone.scm.pipeline.creation.interval=30s
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
 OZONE-SITE.XML_ozone.scm.ec.pipeline.minimum=1

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot

@@ -101,3 +101,23 @@ Get key names from output
         Append To List    ${key_names}    ${key_name}
     END
     [Return]       ${key_names}
+
+Get chunk-info block sizes by group
+    ${output} =    Execute          ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/${TESTFILE} | jq -c '[.keyLocations[] | [.[] | {i: .replicaIndex, s: .blockData.size}] | sort_by(.i) | map(.s)]'
+    [Return]       ${output}
+
+Verify chunk-info block sizes
+    [Arguments]    ${expected_json}
+    ${actual_json} =    Get chunk-info block sizes by group
+    ${actual} =    Evaluate    json.dumps(json.loads('''${actual_json}'''.strip()))    json, json
+    ${expected} =    Evaluate    json.dumps(json.loads('''${expected_json}'''))    json, json
+    Should Be Equal As Strings    ${actual}    ${expected}
+
+Create EC key
+    [Arguments]    ${ec_data}    ${ec_parity}    ${file_size}
+    Execute    dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=1 count=${file_size}
+    Execute    ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${ec_data}-${ec_parity}-1024k -t EC
+
+Create Volume Bucket
+    Execute             ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME}
+    Execute             ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec3-2.robot

@@ -30,20 +30,32 @@ ${TESTFILE}         testfile
 ${EC_DATA}          3
 ${EC_PARITY}        2
 ${OM_SERVICE_ID}    %{OM_SERVICE_ID}
+# single-block stripe: one full data block (1048576), other data blocks=0, parity blocks mirrors data1
+${EC32_SINGLE_BLOCK_STRIPE_SIZES}        [[1048576,0,0,1048576,1048576]]
+# 3 MiB full stripe (3*1048576): all 5 replicas are one full 1024k chunk
+${EC32_FULL_STRIPE_SIZES}           [[1048576,1048576,1048576,1048576,1048576]]
+# group0: full stripe; group1: partial-block stripe with one 1000000 B data block
+${EC32_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES}    [[1048576,1048576,1048576,1048576,1048576],[1000000,0,0,1000000,1000000]]
+# multi-block stripe: data1-2=1048576, data3=2500000%1048576=402848, parity mirrors data1
+${EC32_MULTI_BLOCK_STRIPE_SIZES}    [[1048576,1048576,402848,1048576,1048576]]
 
-*** Keywords ***
-Create Volume Bucket
-    Execute             ozone sh volume create o3://${OM_SERVICE_ID}/${VOLUME}
-    Execute             ozone sh bucket create o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}
+*** Test Cases ***
+Test ozone debug replicas chunk-info single-block stripe
+    # 1*1048576: one full data block in a single-block stripe
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    1048576
+    Verify chunk-info block sizes    ${EC32_SINGLE_BLOCK_STRIPE_SIZES}
 
-Create EC key
-    [arguments]       ${bs}    ${count}    
+Test ozone debug replicas chunk-info full stripe
+    # 3*1048576: EC_DATA full 1024k chunks = one complete stripe
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    3145728
+    Verify chunk-info block sizes    ${EC32_FULL_STRIPE_SIZES}
 
-    Execute           dd if=/dev/urandom of=${TEMP_DIR}/testfile bs=${bs} count=${count}
-    Execute           ozone sh key put o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile ${TEMP_DIR}/testfile -r rs-${EC_DATA}-${EC_PARITY}-1024k -t EC
+Test ozone debug replicas chunk-info full stripe and partial-block stripe
+    # 3*1048576 + 1000000: one full stripe plus a partial-block stripe (1000000 B data block)
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    4145728
+    Verify chunk-info block sizes    ${EC32_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES}
 
-*** Test Cases ***
-Test ozone debug replicas chunk-info
-    Create EC key     1048576    3
-    ${count} =        Execute           ozone debug replicas chunk-info o3://${OM_SERVICE_ID}/${VOLUME}/${BUCKET}/testfile | jq '[.keyLocations[0][] | select(.file | test("\\\\.block$")) | .file] | length'
-    Should Be Equal As Integers         ${count}          5
+Test ozone debug replicas chunk-info multi-block stripe
+    # 2*1048576 + 402848: two full data blocks plus a 402848 B partial data block
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    2500000
+    Verify chunk-info block sizes    ${EC32_MULTI_BLOCK_STRIPE_SIZES}

hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test ozone Debug CLI for EC(6,3) replicated keys
+Library             OperatingSystem
+Library             Process
+Resource            ../ec/lib.resource
+Resource            ../lib/os.robot
+Resource            ozone-debug-keywords.robot
+Test Timeout        5 minute
+Suite Setup         Run Keywords    Wait Until Keyword Succeeds    2min    10sec    Has Enough Datanodes    9
+...                     AND    Create Volume Bucket
+
+*** Variables ***
+${PREFIX}           ${EMPTY}
+${VOLUME}           cli-debug-ec6-volume${PREFIX}
+${BUCKET}           cli-debug-ec6-bucket
+${TESTFILE}         testfile
+${EC_DATA}          6
+${EC_PARITY}        3
+${OM_SERVICE_ID}    %{OM_SERVICE_ID}
+# single-block stripe: one full data block (1048576), other data blocks=0, parity mirrors data1
+${EC63_SINGLE_BLOCK_STRIPE_SIZES}        [[1048576,0,0,0,0,0,1048576,1048576,1048576]]
+# 6 MiB full stripe (6*1048576): all 9 replicas are one full 1024k chunk
+${EC63_FULL_STRIPE_SIZES}           [[1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576]]
+# group0: full stripe; group1: partial-block stripe with one 1000000 B data block
+${EC63_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES}    [[1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576,1048576],[1000000,0,0,0,0,0,1000000,1000000,1000000]]
+# multi-block stripe: data1-3=1048576, data4=3500000%1048576=354272, data5-6=0, parity mirrors data1
+${EC63_MULTI_BLOCK_STRIPE_SIZES}    [[1048576,1048576,1048576,354272,0,0,1048576,1048576,1048576]]
+
+*** Test Cases ***
+Test ozone debug replicas chunk-info single-block stripe
+    # 1*1048576: one full data block in a single-block stripe
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    1048576
+    Verify chunk-info block sizes    ${EC63_SINGLE_BLOCK_STRIPE_SIZES}
+
+Test ozone debug replicas chunk-info full stripe
+    # 6*1048576: EC_DATA full 1024k chunks = one complete stripe
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    6291456
+    Verify chunk-info block sizes    ${EC63_FULL_STRIPE_SIZES}
+
+Test ozone debug replicas chunk-info full stripe and partial-block stripe
+    # 6*1048576 + 1000000: one full stripe plus a partial-block stripe (1000000 B data block)
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    7291456
+    Verify chunk-info block sizes    ${EC63_FULL_AND_PARTIAL_BLOCK_STRIPE_SIZES}
+
+Test ozone debug replicas chunk-info multi-block stripe
+    # 3*1048576 + 354272: three full data blocks plus a 354272 B partial data block
+    Create EC key     ${EC_DATA}    ${EC_PARITY}    3500000
+    Verify chunk-info block sizes    ${EC63_MULTI_BLOCK_STRIPE_SIZES}

hadoop-ozone/dist/src/main/smoketest/ec/awss3ecstorage.robot

@@ -18,6 +18,7 @@ Documentation       S3 gateway test with aws cli with STANDARD_IA storage class
 Library             OperatingSystem
 Library             String
 Resource            ../commonlib.robot
+Resource            lib.resource
 Resource            ../s3/commonawslib.robot
 Resource            ../s3/mpu_lib.robot
 Resource            ../ozone-lib/shell.robot
@@ -34,16 +35,6 @@ Setup EC Multipart Tests
 Teardown EC Multipart Tests
     Remove Files    /tmp/1mb
 
-Count Datanodes In Service
-    ${actual} =    Execute    ozone admin datanode list --node-state HEALTHY --operational-state IN_SERVICE --json | jq -r 'length'
-    [return]       ${actual}
-
-Has Enough Datanodes
-    [arguments]    ${expected}
-    ${actual} =    Count Datanodes In Service
-    Should Be True    ${expected} <= ${actual}
-
-
 *** Variables ***
 ${ENDPOINT_URL}       http://s3g:9878
 ${BUCKET}             generated

hadoop-ozone/dist/src/main/smoketest/ec/lib.resource

@@ -24,6 +24,15 @@ Suite Setup         Get Security Enabled From Config
 ${SCM}       scm
 
 *** Keywords ***
+Count Datanodes In Service
+    ${actual} =    Execute    ozone admin datanode list --node-state HEALTHY --operational-state IN_SERVICE --json | jq -r 'length'
+    [return]       ${actual}
+
+Has Enough Datanodes
+    [arguments]    ${expected}
+    ${actual} =    Count Datanodes In Service
+    Should Be True    ${expected} <= ${actual}
+
 Prepare For Tests
     Execute             dd if=/dev/urandom of=/tmp/1mb bs=1048576 count=1
     Execute             dd if=/dev/urandom of=/tmp/2mb bs=1048576 count=2