Skip to content

Commit

Permalink
Merge pull request #351 from pagopa/develop
Browse files Browse the repository at this point in the history
fix: Promotion to master: mask pan and other fixes
  • Loading branch information
and-mora authored Feb 20, 2023
2 parents 2318345 + 7d27cce commit 4d0c779
Show file tree
Hide file tree
Showing 34 changed files with 333 additions and 168 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @GiovanniMancini @and-mora @TommasoLencioni @petretiandrea
* @GiovanniMancini @and-mora @TommasoLencioni @petretiandrea @lucaconsalvi
4 changes: 2 additions & 2 deletions api/batch/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
<parent>
<artifactId>rtd-ms-transaction-filter-api</artifactId>
<groupId>it.gov.pagopa.rtd.ms.transaction_filter.api</groupId>
<version>2.0.0</version>
<version>2.0.1</version>
</parent>

<artifactId>rtd-ms-transaction-filter-api-batch</artifactId>
<version>2.0.0</version>
<version>2.0.1</version>

<dependencies>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package it.gov.pagopa.rtd.transaction_filter.batch.config;

import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicy;
import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicyImpl;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class AppConfig {

@Bean
public TransactionMaskPolicy getTransactionMaskPolicy() {
return new TransactionMaskPolicyImpl();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import it.gov.pagopa.rtd.transaction_filter.batch.step.tasklet.TransactionChecksumTasklet;
import it.gov.pagopa.rtd.transaction_filter.batch.step.tasklet.TransactionSenderRestTasklet;
import it.gov.pagopa.rtd.transaction_filter.batch.step.writer.ChecksumHeaderWriter;
import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicy;
import it.gov.pagopa.rtd.transaction_filter.connector.FileReportRestClient;
import it.gov.pagopa.rtd.transaction_filter.connector.HpanRestClient;
import it.gov.pagopa.rtd.transaction_filter.connector.HpanRestClient.SasScope;
Expand Down Expand Up @@ -170,6 +171,7 @@ public class TransactionFilterStep {
private ExecutorService writerExecutor;

private PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
private final TransactionMaskPolicy maskPolicy;

/**
* @return instance of the LineTokenizer to be used in the itemReader configured for the job
Expand Down Expand Up @@ -321,7 +323,7 @@ public Partitioner outputRtdFilesPartitioner(StoreService storeService) {
MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
// do not match every file in output directory but only the ones generated from the input file
String outputFileRegex = getOutputFilesRegex(storeService.getTargetInputFile(), RTD_OUTPUT_FILE_PREFIX);
String pathMatcher = outputDirectoryPath + File.separator + outputFileRegex;
String pathMatcher = resolver.getResource(outputDirectoryPath).getURI() + File.separator + outputFileRegex;
partitioner.setResources(resolver.getResources(pathMatcher));
partitioner.partition(partitionerSize);
return partitioner;
Expand Down Expand Up @@ -410,7 +412,7 @@ public Partitioner outputAdeFilesPartitioner(StoreService storeService) {
MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
// do not match every file in output directory but only the ones generated from the input file
String outputFileRegex = getOutputFilesRegex(storeService.getTargetInputFile(), ADE_OUTPUT_FILE_PREFIX);
String pathMatcher = outputDirectoryPath + File.separator + outputFileRegex;
String pathMatcher = resolver.getResource(outputDirectoryPath).getURI() + File.separator + outputFileRegex;
partitioner.setResources(resolver.getResources(pathMatcher));
partitioner.partition(partitionerSize);
return partitioner;
Expand Down Expand Up @@ -737,7 +739,7 @@ public TransactionReaderStepListener transactionAdeStepListener(
@Bean
public TransactionItemReaderListener transactionItemReaderListener(
TransactionWriterService transactionWriterService, String executionDate) {
TransactionItemReaderListener transactionItemReaderListener = new TransactionItemReaderListener();
TransactionItemReaderListener transactionItemReaderListener = new TransactionItemReaderListener(maskPolicy);
transactionItemReaderListener.setExecutionDate(executionDate);
transactionItemReaderListener.setTransactionWriterService(transactionWriterService);
transactionItemReaderListener.setErrorTransactionsLogsPath(transactionLogsPath);
Expand All @@ -752,7 +754,7 @@ public TransactionItemReaderListener transactionItemReaderListener(
@Bean
public TransactionItemReaderListener transactionAdeItemReaderListener(
TransactionWriterService transactionWriterService, String executionDate) {
TransactionItemReaderListener transactionItemReaderListener = new TransactionItemReaderListener();
TransactionItemReaderListener transactionItemReaderListener = new TransactionItemReaderListener(maskPolicy);
transactionItemReaderListener.setExecutionDate(executionDate);
transactionItemReaderListener.setTransactionWriterService(transactionWriterService);
transactionItemReaderListener.setErrorTransactionsLogsPath(transactionLogsPath);
Expand All @@ -767,7 +769,7 @@ public TransactionItemReaderListener transactionAdeItemReaderListener(
@Bean
public TransactionItemWriterListener transactionItemWriteListener(
TransactionWriterService transactionWriterService, String executionDate) {
TransactionItemWriterListener transactionItemWriteListener = new TransactionItemWriterListener();
TransactionItemWriterListener transactionItemWriteListener = new TransactionItemWriterListener(maskPolicy);
transactionItemWriteListener.setExecutionDate(executionDate);
transactionItemWriteListener.setTransactionWriterService(transactionWriterService);
transactionItemWriteListener.setErrorTransactionsLogsPath(transactionLogsPath);
Expand All @@ -782,7 +784,7 @@ public TransactionItemWriterListener transactionItemWriteListener(
@Bean
public TransactionItemProcessListener transactionItemProcessListener(
TransactionWriterService transactionWriterService, String executionDate) {
TransactionItemProcessListener transactionItemProcessListener = new TransactionItemProcessListener();
TransactionItemProcessListener transactionItemProcessListener = new TransactionItemProcessListener(maskPolicy);
transactionItemProcessListener.setExecutionDate(executionDate);
transactionItemProcessListener.setErrorTransactionsLogsPath(transactionLogsPath);
transactionItemProcessListener.setEnableAfterProcessLogging(enableAfterProcessLogging);
Expand All @@ -798,7 +800,7 @@ public TransactionItemProcessListener transactionItemProcessListener(
@Bean
public TransactionItemProcessListener transactionAdeItemProcessListener(
TransactionWriterService transactionWriterService, String executionDate) {
TransactionItemProcessListener transactionItemProcessListener = new TransactionItemProcessListener();
TransactionItemProcessListener transactionItemProcessListener = new TransactionItemProcessListener(maskPolicy);
transactionItemProcessListener.setExecutionDate(executionDate);
transactionItemProcessListener.setErrorTransactionsLogsPath(transactionLogsPath);
transactionItemProcessListener.setEnableAfterProcessLogging(enableAfterProcessLogging);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,25 @@
package it.gov.pagopa.rtd.transaction_filter.batch.step.listener;

import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction;
import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicy;
import it.gov.pagopa.rtd.transaction_filter.service.TransactionWriterService;
import java.util.Optional;
import javax.validation.ConstraintViolationException;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.ItemProcessListener;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.lang.NonNull;
import org.springframework.lang.Nullable;

import java.util.Optional;

/**
* Implementation of {@link ItemProcessListener}, to be used to log and/or store records
* filtered or that have produced an error during a record processing phase
*/
@Slf4j
@Data
@RequiredArgsConstructor
public class TransactionItemProcessListener implements ItemProcessListener<InboundTransaction, InboundTransaction> {

private String errorTransactionsLogsPath;
Expand All @@ -29,13 +32,14 @@ public class TransactionItemProcessListener implements ItemProcessListener<Inbou
private String prefix;
private TransactionWriterService transactionWriterService;
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
private final TransactionMaskPolicy maskPolicy;

@Override
public void beforeProcess(InboundTransaction inboundTransaction) {
public void beforeProcess(@NonNull InboundTransaction inboundTransaction) {
// do nothing
}

public void afterProcess(InboundTransaction item, @Nullable InboundTransaction result) {
public void afterProcess(@NonNull InboundTransaction item, @Nullable InboundTransaction result) {

if (Boolean.TRUE.equals(enableAfterProcessLogging)) {
if (result == null) {
Expand All @@ -46,6 +50,7 @@ public void afterProcess(InboundTransaction item, @Nullable InboundTransaction r
}

if (Boolean.TRUE.equals(enableAfterProcessFileLogging) && result == null) {
maskPolicy.apply(item);
logItemIntoFilteredRecordsFile(item);
}

Expand Down Expand Up @@ -88,6 +93,7 @@ private void logItemIntoFilteredRecordsFile(InboundTransaction item) {
}

public void onProcessError(InboundTransaction item, Exception throwable) {
maskPolicy.apply(item);

if (Boolean.TRUE.equals(transactionWriterService.hasErrorHpan(item.getFilename()
.concat(String.valueOf(item.getLineNumber()))))) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package it.gov.pagopa.rtd.transaction_filter.batch.step.listener;

import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction;
import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicy;
import it.gov.pagopa.rtd.transaction_filter.service.TransactionWriterService;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.ItemReadListener;
import org.springframework.batch.item.file.FlatFileParseException;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.lang.NonNull;


/**
Expand All @@ -17,6 +20,7 @@

@Slf4j
@Data
@RequiredArgsConstructor
public class TransactionItemReaderListener implements ItemReadListener<InboundTransaction> {

private String errorTransactionsLogsPath;
Expand All @@ -28,13 +32,14 @@ public class TransactionItemReaderListener implements ItemReadListener<InboundTr
private TransactionWriterService transactionWriterService;
private String prefix;
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
private final TransactionMaskPolicy maskPolicy;

@Override
public void beforeRead() {
// do nothing
}

public void afterRead(InboundTransaction item) {
public void afterRead(@NonNull InboundTransaction item) {

if (Boolean.TRUE.equals(enableAfterReadLogging)) {
if (loggingFrequency > 1 && item.getLineNumber() % loggingFrequency == 0) {
Expand All @@ -48,7 +53,7 @@ public void afterRead(InboundTransaction item) {
}

@SneakyThrows
public void onReadError(Exception throwable) {
public void onReadError(@NonNull Exception throwable) {

if (Boolean.TRUE.equals(enableOnErrorLogging)) {
log.error("Error while reading a transaction record - {} - {}", throwable.getMessage(),
Expand All @@ -61,7 +66,7 @@ public void onReadError(Exception throwable) {
.replace("]","").replace("\\\\", "/");
String[] fileArr = filename.split("/");
try {
String lineContent = flatFileParseException.getInput();
String lineContent = maskPolicy.apply(flatFileParseException.getInput());
transactionWriterService.write(resolver.getResource(errorTransactionsLogsPath)
.getFile().getAbsolutePath()
.concat("/".concat(executionDate))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package it.gov.pagopa.rtd.transaction_filter.batch.step.listener;

import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction;
import it.gov.pagopa.rtd.transaction_filter.batch.utils.TransactionMaskPolicy;
import it.gov.pagopa.rtd.transaction_filter.service.TransactionWriterService;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -28,6 +29,7 @@ public class TransactionItemWriterListener implements ItemWriteListener<InboundT
private TransactionWriterService transactionWriterService;
private String prefix;
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
private final TransactionMaskPolicy maskPolicy;

@Override
public void beforeWrite(List<? extends InboundTransaction> list) {
Expand Down Expand Up @@ -58,6 +60,7 @@ public void onWriteError(Exception throwable, List<? extends InboundTransaction>
}

if (Boolean.TRUE.equals(enableOnErrorFileLogging)) {
maskPolicy.apply(inboundTransaction);
try {
String filename = inboundTransaction.getFilename().replace("\\", "/");
String[] fileArr = filename.split("/");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package it.gov.pagopa.rtd.transaction_filter.batch.step.tasklet;

import feign.FeignException;
import it.gov.pagopa.rtd.transaction_filter.connector.HpanRestClient;
import it.gov.pagopa.rtd.transaction_filter.connector.SasResponse;
import it.gov.pagopa.rtd.transaction_filter.service.HpanConnectorService;
Expand Down Expand Up @@ -41,7 +42,7 @@ public RepeatStatus execute(StepContribution stepContribution, ChunkContext chun
sasResponse = hpanConnectorService.getSasToken(scope);
hpanConnectorService.uploadFile(resource.getFile(), sasResponse.getSas(), sasResponse.getAuthorizedContainer());
uploadSucceeded = true;
} catch (IOException e) {
} catch (IOException | FeignException e) {
remainingAttempts -= 1;
if (remainingAttempts < 1) {
throw e;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package it.gov.pagopa.rtd.transaction_filter.batch.utils;

import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction;

public interface TransactionMaskPolicy {

/**
* Apply the masking policy to a InboundTransaction object. The object is modified via side effect.
*
* @param transaction object to mask
*/
void apply(InboundTransaction transaction);

/**
* Apply the masking policy to a transaction represented in csv format.
* This method is used when it's not possible to parse the string into a InboundTransaction object.
*
* @param transactionContentAsCsv transaction in csv format with ";" as separator
* @return the string masked
*/
String apply(String transactionContentAsCsv);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package it.gov.pagopa.rtd.transaction_filter.batch.utils;

import it.gov.pagopa.rtd.transaction_filter.batch.model.InboundTransaction;
import lombok.extern.slf4j.Slf4j;
import org.springframework.lang.NonNull;

@Slf4j
public class TransactionMaskPolicyImpl implements TransactionMaskPolicy {

private static final int PAN_INDEX = 3;

@Override
public void apply(InboundTransaction inboundTransaction) {

String maskedPan = maskPan(inboundTransaction.getPan());
inboundTransaction.setPan(maskedPan);
}

@Override
public String apply(@NonNull String transactionContentAsCsv) {

// preliminary check to see if Pan field is found
String[] transactionSplitted = transactionContentAsCsv.split(";", -1);
if (transactionSplitted.length < PAN_INDEX) {
log.debug("It's not possible to apply mask to this string: cannot find the pan field!");
return transactionContentAsCsv;
}

transactionSplitted[PAN_INDEX] = maskPan(transactionSplitted[PAN_INDEX]);

return String.join(";", transactionSplitted);
}

private String maskPan(String pan) {
return pan.replaceAll("(?<=.{6}).(?=.{4})", "*");
}
}
Loading

0 comments on commit 4d0c779

Please sign in to comment.