Reconstruction de Hive dans HDP : patch, test et build

cat hdp_2_6_0_0_patches.txt

HIVE-9941
HIVE-12492
HIVE-14214
...
...
HIVE-16319
HIVE-16323
HIVE-16325

wc -l hdp_2_6_0_0_patches.txt 
120 hdp_2_6_0_0_patches.txt

git clone https://github.com/apache/hive.git
git checkout rel/release-2.1.0

wget https://issues.apache.org/jira/secure/attachment/12833864/HIVE-9941.3.patch

git apply HIVE-9941.3.patch 
HIVE-9941.3.patch:17: new blank line at EOF.
+
HIVE-9941.3.patch:42: new blank line at EOF.
+
HIVE-9941.3.patch:71: new blank line at EOF.
+
HIVE-9941.3.patch:88: new blank line at EOF.
+
warning: 4 lines add whitespace errors.

git status
HEAD detached at rel/release-2.1.0
Untracked files:
  (use "git add <file>..." to include in what will be committed)
	HIVE-9941.3.patch
	ql/src/test/queries/clientnegative/authorization_alter_drop_ptn.q
	ql/src/test/queries/clientnegative/authorization_export_ptn.q
	ql/src/test/queries/clientnegative/authorization_import_ptn.q
	ql/src/test/queries/clientnegative/authorization_truncate_2.q
	ql/src/test/results/clientnegative/authorization_alter_drop_ptn.q.out
	ql/src/test/results/clientnegative/authorization_export_ptn.q.out
	ql/src/test/results/clientnegative/authorization_import_ptn.q.out
	ql/src/test/results/clientnegative/authorization_truncate_2.q.out

nothing added to commit but untracked files present (use "git add" to track)

wget https://issues.apache.org/jira/secure/attachment/12854164/HIVE-12492.02.patch

git apply HIVE-12492.02.patch
HIVE-12492.02.patch:362: trailing whitespace.
        Map 1 
[...]
error: src/java/org/apache/hadoop/hive/conf/HiveConf.java: No such file or directory
error: src/test/resources/testconfiguration.properties: No such file or directory
error: src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java: No such file or directory
error: src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java: No such file or directory
error: src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java: No such file or directory

diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 3777fa9..08422d5 100644
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1422,6 +1422,11 @@ private static void populateLlapDaemonVarsSet(Set<String> llapDaemonVarsSetLocal
         "This controls how many partitions can be scanned for each partitioned table.\n" +
[...]

diff --git a/ql/src/test/queries/clientnegative/authorization_alter_drop_ptn.q b/ql/src/test/queries/clientnegative/authorization_alter_drop_ptn.q

git apply -p0 HIVE-12492.02.patch

HIVE-12492.02.patch:362: trailing whitespace.
        Map 1 
[...]
error: patch failed: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java:1422
error: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java: patch does not apply
error: patch failed: itests/src/test/resources/testconfiguration.properties:501
error: itests/src/test/resources/testconfiguration.properties: patch does not apply
error: patch failed: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java:53
error: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java: patch does not apply
error: patch failed: ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java:51
error: ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java: patch does not apply

generate_patch_manifest = ->
  # For each patch in the given list, return a manifest
  patches_manifest = hdp_2_6_0_0_patches
  # Generate a manifest with patch id, description, creation date and url
  .map (patch) ->
    # Parse the JIRA id
    jira_id = patch.split(/:(.+)/)[0]
    # Parse the commit description
    commit_description = patch.split(/:(.+)/)[1]
    # Get all the attachments
    response = await axios.get "https://issues.apache.org/jira/rest/api/2/search?jql=key=#{jira_id}&fields=attachment"
    # Filter the attachments that are actual patches
    attachments = response.data.issues[0].fields.attachment.filter (o) -> o.filename.includes jira_id
    # Get only the latest patch
    most_recent_created_time = Math.max.apply Math, attachments.map (o) -> Date.parse o.created
    most_recent_created_attachment = attachments.filter (o) -> Date.parse(o.created) is most_recent_created_time
    # Create the patch manifest (implicit return in CoffeeScript)
    id: jira_id
    description: commit_description
    created: Date.parse most_recent_created_attachment[0].created
    url: most_recent_created_attachment[0].content
  # Order the patches by date
  .sort (a,b) ->
    a.created - b.created
  # Write tbe object to a YAML file
  fs.writeFile './patch_manifest.yml', yaml.safeDump(patches_manifest), (err) ->
    if err
    then console.error err 
    else console.info 'done'

- id: $JIRA_NUMBER
  description: $COMMIT_MESSAGE
  created: $TIMESTAMP
  url: $LATEST_PATCH_URL

# For each patch in the manifest
for patch, index in patches_manifest
  try
    await nikita.execute """
    # Download the patch file
    wget -q -N #{patch.url}

    # If the patch as the "a/" leading part in the files, apply with -p1
    if grep "git \\\\ba/" patches/#{patch.url.split('/').pop()}; then
      git apply patches/#{patch.url.split('/').pop()}

    # If not, then apply with -p0
    else
      git apply -p0 patches/#{patch.url.split('/').pop()}
    fi

    # Commit changes
    git add -A
    git commit -m "Applied #{patch.url.split('/').pop()}"
    """
    console.info "Patch #{patch.url.split('/').pop()} applied successfully"
  catch err
    console.error "Patch #{patch.url.split('/').pop()} failed to apply"

git checkout master
git log --pretty=format:'%at;%H;%s' > apache_hive_master_commits.csv

head apache_hive_master_commits.csv

1597212601;a430ac31441ad2d6a03dd24e3141f42c79e022f4;HIVE-23995:Don't set location for managed tables in case of replication (Aasha Medhi, reviewed by Pravin Kumar Sinha)
1597206125;ef47b9eb288dec6e6a014dd5e52accdf0ac3771f;HIVE-24030: Upgrade ORC to 1.5.10 (#1393)
1597144690;d4af3840f89408edea886a28ee4ae7d79a6f16f8;HIVE-24014: Need to delete DumpDirectoryCleanerTask (Arko Sharma, reviewed by Aasha Medhi)
1597135126;71c9af7b8ead470520a6c3a4848be9c67eb80f10;HIVE-24001: Don't cache MapWork in tez/ObjectCache during query-based compaction (Karen Coppage, reviewed by Marta Kuczora)
1597108110;af911c9fe0ff3bb49d80f6c0109d30f5e1046849;HIVE-23996: Remove unused line in UDFArgumentException (Guo Philipse reviewed by Peter Vary, Jesus Camacho Rodriguez)
1597107811;17ee33f4f4d8dc9437d1d3a47663a635d2e47b58;HIVE-23997: Some logs in ConstantPropagateProcFactory are not straightforward (Zhihua Deng, reviewed by Jesus Camacho Rodriguez)
1596833809;5d9a5cf5a36c1d704d2671eb57547ea50249f28b;HIVE-24011: Flaky test AsyncResponseHandlerTest ( Mustafa Iman via Ashutosh Chauhan)
1596762366;9fc8da6f32b68006c222ccfd038719fc89ff8550;HIVE-24004: Improve performance for filter hook for superuser path(Sam An, reviewed by Naveen Gangam)
1595466890;7293956fc2f13ccbe72825b67ce1b53dce536359;HIVE-23901: Overhead of Logger in ColumnStatsMerger damage the performance
1593631609;4457c3ec9360650be021ea84ed1d5d0f007d8308;HIVE-22934 Hive server interactive log counters to error stream ( Ramesh Kumar via Ashutosh Chauhan)

cat hdp_2_6_0_0_patches.txt | while read in; do grep $in apache_hive_master_commits.csv; done > commits_to_apply.csv
sort commits_to_apply.csv > sorted_commits_to_apply.csv

head -1 sorted_commits_to_apply.csv
1469348583;b28ec7fdd8317b47973c6c8f7cdfe805dc20a806;HIVE-14214: ORC Schema Evolution and Predicate Push Down do not work together (no rows returned) (Matt McCline, reviewed by Prasanth Jayachandran/Owen O'Malley)

git cherry-pick -x 15dd7f19bc49ee7017fa5bdd65f9b4ec4dd019d2 --strategy-option theirs
CONFLICT (modify/delete): ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out deleted in HEAD and modified in 15dd7f19bc... HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong). Version 15dd7f19bc... HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong) of ql/src/test/results/clientpositive/tez/explainanalyze_5.q.out left in tree.
[...]
CONFLICT (modify/delete): ql/src/test/queries/clientpositive/explainanalyze_1.q deleted in HEAD and modified in 15dd7f19bc... HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong). Version 15dd7f19bc... HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong) of ql/src/test/queries/clientpositive/explainanalyze_1.q left in tree.
error: could not apply 15dd7f19bc... HIVE-15084: Flaky test: TestMiniTezCliDriver:explainanalyze_1, 2, 3, 4, 5 (Pengcheng Xiong)

Unmerged paths:
  (use "git add/rm ..." as appropriate to mark resolution)
	deleted by us:   llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/SerDeEncodedDataReader.java
	added by them:   llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/VectorDeserializeOrcWriter.java

git checkout rel/release-2.1.0

while read line; do
  commit=$(echo $line | awk -F "\"*;\"*" '{print $2}')
  if git cherry-pick -x $commit --strategy-option theirs > patches/logs/cherry_pick_$commit.out 2>&1; then
    echo "Cherry picking $commit SUCCEEDED"
  else
    git status > patches/logs/cherry_pick_$commit.out 2>&1
    git status | sed -n 's/.* by .*://p' | xargs git add
    git -c core.editor=true cherry-pick --continue
    echo "Cherry picking $commit SUCEEDED with complications"
  fi
done < sorted_commits_to_apply.csv

mvn clean package -Pdist -DskipTests

[ERROR] /home/leo/Apache/hive/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java:[106,5] cannot find symbol
  symbol:   class Timestamp
  location: class org.apache.hadoop.hive.ql.exec.vector.TestStructColumnVector

git clone http://web.archive.org/web/20200906030301/https://github.com/hortonworks/hive2-release/.git
cd hive2-release && ls

README.md

git checkout HDP-2.6.0.3-8-tag

git log --pretty=format:'%at;%H;%s' | grep HIVE-16245
1489790314;2a9150604edfbc2f44eb57104b4be6ac3ed084c7;BUG-77383 backport HIVE-16245: Vectorization: Does not handle non-column key expressions in MERGEPARTIAL mode

git diff 2a9150604edfbc2f44eb57104b4be6ac3ed084c7^ | grep diff

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatch.java