Skip to content

Commit bc11cea

Browse files
anwar6953anwar6953
authored
Merge pull request #823 from caskdata/release/4.3
Merging release/4.3 to develop
2 parents 1ee62d4 + a2c5365 commit bc11cea

11 files changed

+797
-272
lines changed

‎integration-test-remote/pom.xml

+21
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,28 @@
2727

2828
<artifactId>integration-test-remote</artifactId>
2929

30+
<properties>
31+
<hadoop.version>2.3.0</hadoop.version>
32+
<sentry.version>1.7.0</sentry.version>
33+
<cdap.sentry.extn.version>0.6.0</cdap.sentry.extn.version>
34+
</properties>
35+
3036
<dependencies>
37+
<dependency>
38+
<groupId>co.cask.cdap</groupId>
39+
<artifactId>cdap-sentry-model</artifactId>
40+
<version>${cdap.sentry.extn.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>co.cask.cdap</groupId>
44+
<artifactId>cdap-sentry-binding</artifactId>
45+
<version>${cdap.sentry.extn.version}</version>
46+
</dependency>
47+
<dependency>
48+
<groupId>co.cask.cdap</groupId>
49+
<artifactId>cdap-sentry-policy</artifactId>
50+
<version>${cdap.sentry.extn.version}</version>
51+
</dependency>
3152
<dependency>
3253
<groupId>co.cask.cdap.test</groupId>
3354
<artifactId>integration-test-core</artifactId>

‎integration-test-remote/src/test/java/co/cask/cdap/apps/explore/ExploreTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import co.cask.cdap.proto.QueryStatus;
3131
import co.cask.cdap.proto.StreamProperties;
3232
import co.cask.cdap.proto.id.StreamId;
33+
import co.cask.cdap.security.spi.authorization.UnauthorizedException;
3334
import co.cask.cdap.test.ApplicationManager;
3435
import co.cask.cdap.test.AudiTestBase;
3536
import co.cask.cdap.test.FlowManager;
@@ -668,7 +669,7 @@ private List<List<Object>> executionResult2Rows(ExploreExecutionResult execution
668669
}
669670

670671
private void assertStreamEvents(StreamId streamId, String... expectedEvents)
671-
throws UnauthenticatedException, IOException, StreamNotFoundException {
672+
throws UnauthenticatedException, IOException, StreamNotFoundException, UnauthorizedException {
672673

673674
List<StreamEvent> streamEvents = Lists.newArrayList();
674675
getStreamClient().getEvents(streamId, 0, Long.MAX_VALUE, Integer.MAX_VALUE, streamEvents);

‎integration-test-remote/src/test/java/co/cask/cdap/security/AuthorizationTestBase.java

+264-1
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,72 @@
1616

1717
package co.cask.cdap.security;
1818

19+
import co.cask.cdap.client.AuthorizationClient;
1920
import co.cask.cdap.client.NamespaceClient;
2021
import co.cask.cdap.client.config.ClientConfig;
2122
import co.cask.cdap.client.util.RESTClient;
2223
import co.cask.cdap.proto.ConfigEntry;
2324
import co.cask.cdap.proto.NamespaceMeta;
25+
import co.cask.cdap.proto.element.EntityType;
26+
import co.cask.cdap.proto.id.ApplicationId;
27+
import co.cask.cdap.proto.id.ArtifactId;
28+
import co.cask.cdap.proto.id.DatasetId;
29+
import co.cask.cdap.proto.id.DatasetModuleId;
30+
import co.cask.cdap.proto.id.DatasetTypeId;
31+
import co.cask.cdap.proto.id.EntityId;
32+
import co.cask.cdap.proto.id.InstanceId;
33+
import co.cask.cdap.proto.id.KerberosPrincipalId;
2434
import co.cask.cdap.proto.id.NamespaceId;
35+
import co.cask.cdap.proto.id.ProgramId;
36+
import co.cask.cdap.proto.id.SecureKeyId;
37+
import co.cask.cdap.proto.id.StreamId;
38+
import co.cask.cdap.proto.security.Action;
39+
import co.cask.cdap.proto.security.Role;
40+
import co.cask.cdap.security.authorization.sentry.model.Application;
41+
import co.cask.cdap.security.authorization.sentry.model.Artifact;
42+
import co.cask.cdap.security.authorization.sentry.model.Authorizable;
43+
import co.cask.cdap.security.authorization.sentry.model.Dataset;
44+
import co.cask.cdap.security.authorization.sentry.model.DatasetModule;
45+
import co.cask.cdap.security.authorization.sentry.model.DatasetType;
46+
import co.cask.cdap.security.authorization.sentry.model.Instance;
47+
import co.cask.cdap.security.authorization.sentry.model.Namespace;
48+
import co.cask.cdap.security.authorization.sentry.model.Program;
49+
import co.cask.cdap.security.authorization.sentry.model.SecureKey;
50+
import co.cask.cdap.security.authorization.sentry.model.Stream;
2551
import co.cask.cdap.test.AudiTestBase;
52+
import com.google.common.annotations.VisibleForTesting;
2653
import com.google.common.base.Preconditions;
54+
import com.google.common.collect.Sets;
2755
import com.google.gson.Gson;
2856
import com.google.gson.GsonBuilder;
57+
import org.apache.hadoop.conf.Configuration;
58+
import org.apache.hadoop.security.UserGroupInformation;
59+
import org.apache.sentry.provider.db.SentryNoSuchObjectException;
60+
import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClient;
61+
import org.apache.sentry.provider.db.generic.service.thrift.SentryGenericServiceClientFactory;
62+
import org.apache.sentry.provider.db.generic.service.thrift.TAuthorizable;
63+
import org.apache.sentry.provider.db.generic.service.thrift.TSentryGrantOption;
64+
import org.apache.sentry.provider.db.generic.service.thrift.TSentryPrivilege;
65+
import org.apache.sentry.service.thrift.ServiceConstants;
2966
import org.junit.Before;
3067

68+
import java.io.IOException;
69+
import java.net.URI;
70+
import java.net.URISyntaxException;
71+
import java.util.ArrayList;
72+
import java.util.HashSet;
73+
import java.util.LinkedList;
74+
import java.util.List;
75+
import java.util.Map;
76+
import java.util.Set;
3177
import javax.annotation.Nullable;
78+
import javax.security.auth.callback.Callback;
79+
import javax.security.auth.callback.CallbackHandler;
80+
import javax.security.auth.callback.NameCallback;
81+
import javax.security.auth.callback.PasswordCallback;
82+
import javax.security.auth.callback.UnsupportedCallbackException;
83+
import javax.security.auth.login.LoginContext;
84+
import javax.security.auth.login.LoginException;
3285

3386
/**
3487
* Authorization test base for all authorization tests
@@ -45,10 +98,33 @@ public abstract class AuthorizationTestBase extends AudiTestBase {
4598
protected static final String VERSION = "1.0.0";
4699
protected static final String NO_PRIVILEGE_MSG = "does not have privileges to access entity";
47100

101+
private static final String COMPONENT = "cdap";
102+
private static final String INSTANCE_NAME = "cdap";
103+
private static final Role DUMMY_ROLE = new Role("dummy");
104+
105+
// TODO: Remove this when we migrate to wildcard privilege
106+
protected Set<EntityId> cleanUpEntities;
107+
private SentryGenericServiceClient sentryClient;
108+
private AuthorizationClient authorizationClient;
109+
48110
// General test namespace
49111
protected NamespaceMeta testNamespace = getNamespaceMeta(new NamespaceId("authorization"), null, null,
50112
null, null, null, null);
51113

114+
@Override
115+
public void setUp() throws Exception {
116+
sentryClient = SentryGenericServiceClientFactory.create(getSentryConfig());
117+
// TODO: remove this once caching in sentry is fixed
118+
ClientConfig adminConfig = getClientConfig(fetchAccessToken(ADMIN_USER, ADMIN_USER));
119+
RESTClient adminClient = new RESTClient(adminConfig);
120+
authorizationClient = new AuthorizationClient(adminConfig, adminClient);
121+
userGrant(ADMIN_USER, NamespaceId.DEFAULT, Action.ADMIN);
122+
invalidateCache();
123+
super.setUp();
124+
userRevoke(ADMIN_USER);
125+
cleanUpEntities = new HashSet<>();
126+
}
127+
52128
@Before
53129
public void setup() throws Exception {
54130
ConfigEntry configEntry = this.getMetaClient().getCDAPConfig().get("security.authorization.enabled");
@@ -57,16 +133,78 @@ public void setup() throws Exception {
57133
Preconditions.checkState(Boolean.parseBoolean(configEntry.getValue()), "Authorization not enabled.");
58134
}
59135

136+
@Override
137+
public void tearDown() throws Exception {
138+
// we have to grant ADMIN privileges to all clean up entites such that these entities can be deleted
139+
for (EntityId entityId : cleanUpEntities) {
140+
userGrant(ADMIN_USER, entityId, Action.ADMIN);
141+
}
142+
userGrant(ADMIN_USER, testNamespace.getNamespaceId(), Action.ADMIN);
143+
userGrant(ADMIN_USER, NamespaceId.DEFAULT, Action.ADMIN);
144+
invalidateCache();
145+
// teardown in parent deletes all entities
146+
super.tearDown();
147+
// reset the test by revoking privileges from all users.
148+
userRevoke(ADMIN_USER);
149+
userRevoke(ALICE);
150+
userRevoke(BOB);
151+
userRevoke(CAROL);
152+
userRevoke(EVE);
153+
sentryClient.close();
154+
}
155+
60156
protected NamespaceId createAndRegisterNamespace(NamespaceMeta namespaceMeta, ClientConfig config,
61-
RESTClient client) throws Exception {
157+
RESTClient client) throws Exception {
62158
try {
63159
new NamespaceClient(config, client).create(namespaceMeta);
160+
cleanUpEntities.add(namespaceMeta.getNamespaceId());
64161
} finally {
65162
registerForDeletion(namespaceMeta.getNamespaceId());
66163
}
67164
return namespaceMeta.getNamespaceId();
68165
}
69166

167+
/**
168+
* Grants action privilege to user on entityId. Creates a role for the user. Grant action privilege
169+
* on that role, and add the role to the group the user belongs to. All done through sentry.
170+
* @param user The user we want to grant privilege to.
171+
* @param entityId The entity we want to grant privilege on.
172+
* @param action The privilege we want to grant.
173+
*/
174+
protected void userGrant(String user, EntityId entityId, Action action) throws Exception {
175+
// create role and add to group
176+
// TODO: use a different user as Sentry Admin (neither CDAP, nor an user used in our tests)
177+
sentryClient.createRoleIfNotExist(ADMIN_USER, user, COMPONENT);
178+
sentryClient.addRoleToGroups(ADMIN_USER, user, COMPONENT, Sets.newHashSet(user));
179+
180+
// create authorizable list
181+
List<TAuthorizable> authorizables = toTAuthorizable(entityId);
182+
TSentryPrivilege privilege = new TSentryPrivilege(COMPONENT, INSTANCE_NAME, authorizables, action.name());
183+
privilege.setGrantOption(TSentryGrantOption.TRUE);
184+
sentryClient.grantPrivilege(ADMIN_USER, user, COMPONENT, privilege);
185+
}
186+
187+
protected void invalidateCache() throws Exception {
188+
// TODO: Hack to invalidate cache in sentry authorizer. Remove once cache problem is solved.
189+
authorizationClient.dropRole(DUMMY_ROLE);
190+
}
191+
192+
/**
193+
* Revokes all privileges from user. Deletes the user role through sentry.
194+
*
195+
* @param user The user we want to revoke privilege from.
196+
*/
197+
protected void userRevoke(String user) throws Exception {
198+
try {
199+
sentryClient.deleteRoleToGroups(ADMIN_USER, user, COMPONENT, Sets.newHashSet(user));
200+
} catch (SentryNoSuchObjectException e) {
201+
// skip a role that hasn't been added to the user
202+
} finally {
203+
sentryClient.dropRoleIfExists(ADMIN_USER, user, COMPONENT);
204+
invalidateCache();
205+
}
206+
}
207+
70208
protected NamespaceMeta getNamespaceMeta(NamespaceId namespaceId, @Nullable String principal,
71209
@Nullable String groupName, @Nullable String keytabURI,
72210
@Nullable String rootDirectory, @Nullable String hbaseNamespace,
@@ -82,4 +220,129 @@ protected NamespaceMeta getNamespaceMeta(NamespaceId namespaceId, @Nullable Stri
82220
.setHiveDatabase(hiveDatabase)
83221
.build();
84222
}
223+
224+
private Configuration getSentryConfig() throws IOException, LoginException, URISyntaxException {
225+
226+
String hostUri = super.getInstanceURI();
227+
String sentryRpcAddr = new URI(hostUri).getHost();
228+
String sentryPrincipal = "sentry/" + sentryRpcAddr + "@CONTINUUITY.NET";
229+
String sentryRpcPort = "8038";
230+
231+
Configuration conf = new Configuration(false);
232+
conf.clear();
233+
conf.set(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS);
234+
conf.set(ServiceConstants.ServerConfig.PRINCIPAL, sentryPrincipal);
235+
conf.set(ServiceConstants.ClientConfig.SERVER_RPC_ADDRESS, sentryRpcAddr);
236+
conf.set(ServiceConstants.ClientConfig.SERVER_RPC_PORT, sentryRpcPort);
237+
238+
// Log in to Kerberos
239+
UserGroupInformation.setConfiguration(conf);
240+
LoginContext lc = kinit();
241+
UserGroupInformation.loginUserFromSubject(lc.getSubject());
242+
return conf;
243+
}
244+
245+
private static LoginContext kinit() throws LoginException {
246+
LoginContext lc = new LoginContext(BasicAuthorizationTest.class.getSimpleName(), new CallbackHandler() {
247+
public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException {
248+
for (Callback c : callbacks) {
249+
if (c instanceof NameCallback) {
250+
((NameCallback) c).setName(ADMIN_USER);
251+
}
252+
if (c instanceof PasswordCallback) {
253+
((PasswordCallback) c).setPassword(ADMIN_USER.toCharArray());
254+
}
255+
}
256+
}
257+
});
258+
lc.login();
259+
return lc;
260+
}
261+
262+
private List<TAuthorizable> toTAuthorizable(EntityId entityId) {
263+
List<org.apache.sentry.core.common.Authorizable> authorizables = toSentryAuthorizables(entityId);
264+
List<TAuthorizable> tAuthorizables = new ArrayList<>();
265+
for (org.apache.sentry.core.common.Authorizable authorizable : authorizables) {
266+
tAuthorizables.add(new TAuthorizable(authorizable.getTypeName(), authorizable.getName()));
267+
}
268+
return tAuthorizables;
269+
}
270+
271+
@VisibleForTesting
272+
List<org.apache.sentry.core.common.Authorizable> toSentryAuthorizables(final EntityId entityId) {
273+
List<org.apache.sentry.core.common.Authorizable> authorizables = new LinkedList<>();
274+
toAuthorizables(entityId, authorizables);
275+
return authorizables;
276+
}
277+
278+
private void toAuthorizables(EntityId entityId, List<? super Authorizable> authorizables) {
279+
EntityType entityType = entityId.getEntityType();
280+
switch (entityType) {
281+
case INSTANCE:
282+
authorizables.add(new Instance(((InstanceId) entityId).getInstance()));
283+
break;
284+
case NAMESPACE:
285+
toAuthorizables(new InstanceId(INSTANCE_NAME), authorizables);
286+
authorizables.add(new Namespace(((NamespaceId) entityId).getNamespace()));
287+
break;
288+
case ARTIFACT:
289+
ArtifactId artifactId = (ArtifactId) entityId;
290+
toAuthorizables(artifactId.getParent(), authorizables);
291+
authorizables.add(new Artifact(artifactId.getArtifact()));
292+
break;
293+
case APPLICATION:
294+
ApplicationId applicationId = (ApplicationId) entityId;
295+
toAuthorizables(applicationId.getParent(), authorizables);
296+
authorizables.add(new Application(applicationId.getApplication()));
297+
break;
298+
case DATASET:
299+
DatasetId dataset = (DatasetId) entityId;
300+
toAuthorizables(dataset.getParent(), authorizables);
301+
authorizables.add(new Dataset(dataset.getDataset()));
302+
break;
303+
case DATASET_MODULE:
304+
DatasetModuleId datasetModuleId = (DatasetModuleId) entityId;
305+
toAuthorizables(datasetModuleId.getParent(), authorizables);
306+
authorizables.add(new DatasetModule(datasetModuleId.getModule()));
307+
break;
308+
case DATASET_TYPE:
309+
DatasetTypeId datasetTypeId = (DatasetTypeId) entityId;
310+
toAuthorizables(datasetTypeId.getParent(), authorizables);
311+
authorizables.add(new DatasetType(datasetTypeId.getType()));
312+
break;
313+
case STREAM:
314+
StreamId streamId = (StreamId) entityId;
315+
toAuthorizables(streamId.getParent(), authorizables);
316+
authorizables.add(new Stream((streamId).getStream()));
317+
break;
318+
case PROGRAM:
319+
ProgramId programId = (ProgramId) entityId;
320+
toAuthorizables(programId.getParent(), authorizables);
321+
authorizables.add(new Program(programId.getType(), programId.getProgram()));
322+
break;
323+
case SECUREKEY:
324+
SecureKeyId secureKeyId = (SecureKeyId) entityId;
325+
toAuthorizables(secureKeyId.getParent(), authorizables);
326+
authorizables.add(new SecureKey(secureKeyId.getName()));
327+
break;
328+
case KERBEROSPRINCIPAL:
329+
KerberosPrincipalId principalId = (KerberosPrincipalId) entityId;
330+
toAuthorizables(new InstanceId(INSTANCE_NAME), authorizables);
331+
authorizables.add(new co.cask.cdap.security.authorization.sentry.model.Principal(principalId.getPrincipal()));
332+
break;
333+
default:
334+
throw new IllegalArgumentException(String.format("The entity %s is of unknown type %s", entityId, entityType));
335+
}
336+
}
337+
338+
// TODO: Remove this when we migrate to wildcard privilege
339+
protected void setUpPrivilegeAndRegisterForDeletion(String user,
340+
Map<EntityId, Set<Action>> neededPrivileges) throws Exception {
341+
for (Map.Entry<EntityId, Set<Action>> privilege : neededPrivileges.entrySet()) {
342+
for (Action action : privilege.getValue()) {
343+
userGrant(user, privilege.getKey(), action);
344+
cleanUpEntities.add(privilege.getKey());
345+
}
346+
}
347+
}
85348
}

0 commit comments

Comments
 (0)