Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ADH-5313] Fix problems during file rename syncing #132

Merged
merged 6 commits into from
Nov 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions conf/smart-default.xml
Original file line number Diff line number Diff line change
Expand Up @@ -425,17 +425,6 @@
</description>
</property>

<property>
<name>smart.sync.schedule.strategy</name>
<value>UNORDERED</value>
<description>
Strategy of copying files during 'sync' rule. Possible values:
FIFO - the files created/modified first will be scheduled for transfer first
LIFO - the files created/modified last will be scheduled for transfer first
UNORDERED - no guarantees of the file scheduling order
</description>
</property>

<property>
<name>smart.sync.file.equality.strategy</name>
<value>CHECKSUM</value>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public class SyncAction extends SmartAction {
// related to remote cluster and fileDiff.src
public static final String DEST = "-dest";
public static final String PRESERVE = "-preserve";
public static final String BASE_OPERATION = "-baseOperation";

@Override
protected void execute() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,6 @@ public class SmartConfKeys {
"smart.dispatch.cmdlets.extra.num";
public static final int SMART_DISPATCH_CMDLETS_EXTRA_NUM_DEFAULT = 10;

public static final String SMART_SYNC_SCHEDULE_STRATEGY_KEY = "smart.sync.schedule.strategy";
public static final String SMART_SYNC_SCHEDULE_STRATEGY_DEFAULT = "UNORDERED";


public static final String SMART_SYNC_FILE_EQUALITY_STRATEGY =
"smart.sync.file.equality.strategy";
public static final String SMART_SYNC_FILE_EQUALITY_STRATEGY_DEFAULT = "CHECKSUM";
Expand Down
122 changes: 6 additions & 116 deletions smart-common/src/main/java/org/smartdata/model/FileDiff.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,16 @@
*/
package org.smartdata.model;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;

import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;

@Data
@AllArgsConstructor
@Builder(toBuilder = true)
public class FileDiff {
private long diffId;
private long ruleId;
Expand All @@ -51,119 +53,7 @@ public FileDiff(FileDiffType diffType, FileDiffState state) {
this.state = state;
}

public long getDiffId() {
return diffId;
}

public void setDiffId(long diffId) {
this.diffId = diffId;
}

public long getRuleId() {
return ruleId;
}

public void setRuleId(long ruleId) {
this.ruleId = ruleId;
}

public FileDiffType getDiffType() {
return diffType;
}

public void setDiffType(FileDiffType diffType) {
this.diffType = diffType;
}

public String getSrc() {
return src;
}

public void setSrc(String src) {
this.src = src;
}

public Map<String, String> getParameters() {
return parameters;
}

public void setParameters(Map<String, String> parameters) {
this.parameters = parameters;
}

public void setParameter(String key, String value) {
parameters.put(key, value);
}

public String getParametersJsonString() {
Gson gson = new Gson();
return gson.toJson(parameters);
}

public void setParametersFromJsonString(String jsonParameters) {
Gson gson = new Gson();
parameters = gson.fromJson(jsonParameters,
new TypeToken<Map<String, String>>() {
}.getType());
}

public String getParametersString() {
StringBuffer ret = new StringBuffer();
if (parameters.containsKey("-dest")) {
ret.append(String.format(" -dest %s", parameters.get("-dest")));
parameters.remove("-dest");
}
for (Iterator<Map.Entry<String, String>> it = parameters.entrySet().iterator(); it.hasNext();) {
Map.Entry<String, String> entry = it.next();
ret.append(String.format(" %s %s", entry.getKey(), entry.getValue()));
}
return ret.toString();
}


public FileDiffState getState() {
return state;
}

public void setState(FileDiffState state) {
this.state = state;
}

public long getCreateTime() {
return createTime;
}

public void setCreateTime(long createTime) {
this.createTime = createTime;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
FileDiff fileDiff = (FileDiff) o;
return diffId == fileDiff.diffId
&& ruleId == fileDiff.ruleId
&& createTime == fileDiff.createTime
&& diffType == fileDiff.diffType
&& Objects.equals(src, fileDiff.src)
&& Objects.equals(parameters, fileDiff.parameters)
&& state == fileDiff.state;
}

@Override
public int hashCode() {
return Objects.hash(diffId, ruleId, diffType, src, parameters, state, createTime);
}

@Override
public String toString() {
return String.format(
"FileDiff{diffId=%s, parameters=%s, src=%s, diffType=%s, state=%s, createTime=%s}",
diffId, parameters, src, diffType, state.getValue(), createTime);
}
}
Loading
Loading