Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gh-3326: Update Schema merge federated POC #3328

Merged
merged 8 commits into from
Oct 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2023 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -71,7 +71,9 @@
* <tr><td> n/a </td><td> {@code empty} </td><td> F
* </td><td> F </td></tr>
* </table>
* @deprecated Will be replaced by a GraphAccess class in 2.4.0.
*/
@Deprecated
@JsonDeserialize(builder = FederatedAccess.Builder.class)
@JsonPropertyOrder(value = {"class", "owningUserId"}, alphabetic = true)
public class FederatedAccess implements AccessControlledResource {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@
import static java.util.Objects.nonNull;
import static uk.gov.gchq.gaffer.accumulostore.utils.TableUtils.renameTable;

/**
* @deprecated Will be removed in 2.4.0.
*/
@Deprecated
public class FederatedGraphStorage {
public static final String ERROR_ADDING_GRAPH_TO_CACHE = "Error adding graph, GraphId is known within the cache, but %s is different. GraphId: %s";
public static final String USER_IS_ATTEMPTING_TO_OVERWRITE = "User is attempting to overwrite a graph within FederatedStore. GraphId: %s";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,14 @@
import uk.gov.gchq.gaffer.jsonserialisation.JSONSerialiser;

/**
* Wrapper around {@link FederatedStoreCacheTransient} to provide an interface for
* Wrapper around {@link FederatedStoreCacheTransient} to provide an interface
* for
* handling {@link Graph}s within a {@link FederatedStore}.
*
* @deprecated Federated store will use the default cache implementation going
* forward.
*/
@Deprecated
public final class FederatedStoreCache {
private final FederatedStoreCacheTransient cacheTransient;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@
/**
* Implementation of {@link Cache} for handling
* {@link Graph}s within a {@link FederatedStore}.
*
* @deprecated Federated store will use the default cache implementation going
* forward.
*/
@Deprecated
public class FederatedStoreCacheTransient extends Cache<String, Pair<GraphSerialisable, byte[]>> {
public static final String ERROR_ADDING_GRAPH_TO_CACHE_GRAPH_ID_S = "Error adding graph to cache. graphId: %s";
private static final String CACHE_SERVICE_NAME_PREFIX = "federatedStoreGraphs";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2022 Crown Copyright
* Copyright 2020-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -27,6 +27,11 @@
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static uk.gov.gchq.gaffer.federatedstore.FederatedStoreConstants.FEDERATED_STORE_SYSTEM_USER;

/**
* @deprecated Marked for removal in 2.4.0 please use standard AccessPredicates
* going forward.
*/
@Deprecated
public abstract class FederatedGraphAccessPredicate extends AccessPredicate {

public FederatedGraphAccessPredicate(final Predicate<User> userPredicate) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Crown Copyright
* Copyright 2020-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,6 +28,11 @@

import static java.util.Collections.emptyList;

/**
* @deprecated Marked for removal in 2.4.0 please use standard AccessPredicates
* going forward.
*/
@Deprecated
public class FederatedGraphReadAccessPredicate extends FederatedGraphAccessPredicate {

public FederatedGraphReadAccessPredicate(final String creatingUserId, final Set<String> auths, final boolean isPublic) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Crown Copyright
* Copyright 2020-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -13,13 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.federatedstore.access.predicate;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

import uk.gov.gchq.gaffer.federatedstore.access.predicate.user.FederatedGraphWriteUserPredicate;

/**
* @deprecated Marked for removal in 2.4.0 please use standard AccessPredicates
* going forward.
*/
@Deprecated
public class FederatedGraphWriteAccessPredicate extends FederatedGraphAccessPredicate {

@JsonCreator
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2021 Crown Copyright
* Copyright 2020-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.federatedstore.access.predicate.user;

import com.fasterxml.jackson.annotation.JsonCreator;
Expand All @@ -27,6 +28,11 @@

import static java.util.Objects.isNull;

/**
* @deprecated Marked for removal in 2.4.0 please use standard
* DefaultUserPredicate going forward.
*/
@Deprecated
public class FederatedGraphReadUserPredicate extends DefaultUserPredicate {

private final boolean isPublic;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020 Crown Copyright
* Copyright 2020-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package uk.gov.gchq.gaffer.federatedstore.access.predicate.user;

import com.fasterxml.jackson.annotation.JsonCreator;
Expand All @@ -26,6 +27,11 @@

import static java.util.Collections.emptyList;

/**
* @deprecated Marked for removal in 2.4.0 please use standard
* DefaultUserPredicate going forward.
*/
@Deprecated
public class FederatedGraphWriteUserPredicate extends DefaultUserPredicate {

@JsonCreator
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2018-2023 Crown Copyright
* Copyright 2018-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -23,6 +23,11 @@
import uk.gov.gchq.koryphe.Since;
import uk.gov.gchq.koryphe.Summary;

/**
* @deprecated Will be removed in 2.4.0, functionality will be merged into the
* AddGraph operation.
*/
@Deprecated
@JsonPropertyOrder(
value = {"class", "graphId"},
alphabetic = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,15 @@
import static uk.gov.gchq.gaffer.federatedstore.util.FederatedStoreUtil.getCleanStrings;

/**
* This operation federates a payload operation across a given set of graphs and merges the results with a given function.
* This operation federates a payload operation across a given set of graphs and
* merges the results with a given function.
*
* @param <INPUT> Input type of the payload operation
* @param <OUTPUT> Output type of the merge function
* @deprecated Concept of a FederatedOperation class will not exist from 2.4.0,
* all federation specifics are handled via operation options.
*/
@Deprecated
@JsonPropertyOrder(value = {"class", "operation", "mergeFunction", "graphIds", "skipFailedFederatedExecution"}, alphabetic = true)
@Since("2.0.0")
@Summary("Federates a payload operation across given graphs and merges the results with a given function.")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2021-2022 Crown Copyright
* Copyright 2021-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -25,7 +25,11 @@

/**
* Interface for Operations that uses a selection of graphs to be performed.
*
* @deprecated Concept of a FederatedOperation class will not exist from 2.4.0,
* all federation specifics are handled via operation options.
*/
@Deprecated
@Since("2.0.0")
public interface IFederatedOperation extends Operation {

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2021-2023 Crown Copyright
* Copyright 2021-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -24,9 +24,15 @@
import uk.gov.gchq.koryphe.Since;

/**
* {@link IFederationOperation} interface is for special operations used to configure/manipulate/control federation.
* It has no intended function outside of federation and should only be handled by the {@link uk.gov.gchq.gaffer.federatedstore.FederatedStore}.
* {@link IFederationOperation} interface is for special operations used to
* configure/manipulate/control federation.
* It has no intended function outside of federation and should only be handled
* by the {@link uk.gov.gchq.gaffer.federatedstore.FederatedStore}.
*
* @deprecated Concept of a FederatedOperation class will not exist from 2.4.0,
* all federation specifics are handled via operation options.
*/
@Deprecated
@Since("2.0.0")
public interface IFederationOperation extends Operation {

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2017-2022 Crown Copyright
* Copyright 2017-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -22,6 +22,11 @@
import uk.gov.gchq.koryphe.Since;
import uk.gov.gchq.koryphe.Summary;

/**
* @deprecated Will be removed in 2.4.0, functionality will be merged into the
* RemoveGraph operation.
*/
@Deprecated
@JsonPropertyOrder(value = {"class", "graphId"}, alphabetic = true)
@Since("2.0.0")
@Summary("Used to tell a graph to delete all data, before being removed.")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2022 Crown Copyright
* Copyright 2022-2024 Crown Copyright
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -29,7 +29,9 @@
* @param <U> – the type of the second argument to the function
* @param <R> – the type of the result of the function
* @see BiFunction
* @deprecated Merging will be overhauled in 2.4.0.
*/
@Deprecated
public interface ContextSpecificMergeFunction<T, U, R> extends BiFunction<T, U, R> {
ContextSpecificMergeFunction<T, U, R> createFunctionWithContext(final HashMap<String, Object> context) throws GafferCheckedException;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,16 @@ public Schema getSchema(final List<GraphSerialisable> graphs) {
}
}

/**
* Access to getting the operations that have handlers specific to this
* store e.g. operations that should always be handled specially.
*
* @return The Operation classes handled by this store.
*/
public Set<Class<? extends Operation>> getStoreSpecificOperations() {
return storeHandlers.keySet();
}

@Override
public void initialise(final String graphId, final Schema unused, final StoreProperties properties) throws StoreException {
super.initialise(graphId, new Schema(), properties);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,23 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import uk.gov.gchq.gaffer.federated.simple.FederatedStore;
import uk.gov.gchq.gaffer.operation.Operation;
import uk.gov.gchq.gaffer.operation.OperationChain;
import uk.gov.gchq.gaffer.operation.OperationException;
import uk.gov.gchq.gaffer.store.Context;
import uk.gov.gchq.gaffer.store.Store;
import uk.gov.gchq.gaffer.store.operation.handler.OperationChainHandler;
import uk.gov.gchq.gaffer.store.operation.handler.OperationHandler;
import uk.gov.gchq.gaffer.store.operation.handler.named.AddToCacheHandler;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
* Custom handler for operations that could in theory target sub graphs or the
* federated store directly. Implements the {@link AddToCacheHandler} interface
* federated store directly. Implements the {@link AddToCacheHandler} interface
* so that this can also handle operations that add named operations etc.
*/
public class EitherOperationHandler<O extends Operation> implements AddToCacheHandler<O> {
Expand All @@ -49,6 +56,18 @@ public EitherOperationHandler(final OperationHandler<O> standardHandler) {
public Object doOperation(final O operation, final Context context, final Store store) throws OperationException {
LOGGER.debug("Checking if Operation should be handled locally or on sub graphs: {}", operation);

// Check inside operation chain for if all the operations are handled by a federated store
if (operation instanceof OperationChain) {
Set<Class<? extends Operation>> storeSpecificOps = ((FederatedStore) store).getStoreSpecificOperations();
List<Class<? extends Operation>> chainOps = ((OperationChain<?>) operation).flatten().stream()
.map(Operation::getClass)
.collect(Collectors.toList());
if (storeSpecificOps.containsAll(chainOps)) {
return new OperationChainHandler<>(store.getOperationChainValidator(), store.getOperationChainOptimisers())
.doOperation((OperationChain<Object>) operation, context, store);
}
}

// If we have graph IDs then run as a federated operation
if (operation.containsOption(FederatedOperationHandler.OPT_GRAPH_IDS) ||
operation.containsOption(FederatedOperationHandler.OPT_SHORT_GRAPH_IDS) ||
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,11 @@
if (!Boolean.parseBoolean(operation.getOption(OPT_SKIP_FAILED_EXECUTE, "false"))) {
throw e;
}
LOGGER.info("Continuing operation execution on sub graphs");

Check warning on line 122 in store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java

View check run for this annotation

Codecov / codecov/patch

store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOperationHandler.java#L122

Added line #L122 was not covered by tests
} catch (final IllegalArgumentException e) {
// An operation may fail validation for a sub graph this is not really an error.
// We can just continue to execute on the rest of the graphs
LOGGER.warn("Operation contained invalid arguments for a sub graph, skipped execution on graph: {}", gs.getGraphId());
}

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@
if (!Boolean.parseBoolean(operation.getOption(OPT_SKIP_FAILED_EXECUTE, "false"))) {
throw e;
}
LOGGER.info("Continuing operation execution on sub graphs");
} catch (final IllegalArgumentException e) {

Check warning on line 65 in store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java

View check run for this annotation

Codecov / codecov/patch

store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java#L65

Added line #L65 was not covered by tests
// An operation may fail validation for a sub graph this is not really an error.
// We can just continue to execute on the rest of the graphs
LOGGER.warn("Operation contained invalid arguments for a sub graph, skipped execution on graph: {}", gs.getGraphId());

Check warning on line 68 in store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java

View check run for this annotation

Codecov / codecov/patch

store-implementation/simple-federated-store/src/main/java/uk/gov/gchq/gaffer/federated/simple/operation/handler/FederatedOutputHandler.java#L68

Added line #L68 was not covered by tests
}
}

Expand Down
Loading
Loading