diff --git a/src/main/java/de/thetaphi/forbiddenapis/ClassScanner.java b/src/main/java/de/thetaphi/forbiddenapis/ClassScanner.java
index 065c432..33b55f0 100644
--- a/src/main/java/de/thetaphi/forbiddenapis/ClassScanner.java
+++ b/src/main/java/de/thetaphi/forbiddenapis/ClassScanner.java
@@ -87,7 +87,7 @@ public String getSourceFile() {
     return source;
   }
   
-  String checkClassUse(Type type, String what, boolean deep, String origInternalName) {
+  String checkClassUse(Type type, String what, boolean isAnnotation, String origInternalName) {
     while (type.getSort() == Type.ARRAY) {
       type = type.getElementType(); // unwrap array
     }
@@ -98,7 +98,8 @@ String checkClassUse(Type type, String what, boolean deep, String origInternalNa
     if (violation != null) {
       return violation;
     }
-    if (deep && forbidNonPortableRuntime) {
+    // try best to check for non portable runtime
+    if (forbidNonPortableRuntime) try {
       final String binaryClassName = type.getClassName();
       final ClassMetadata c = lookup.lookupRelatedClass(type.getInternalName(), origInternalName);
       if (c != null && c.isNonPortableRuntime) {
@@ -107,12 +108,15 @@ String checkClassUse(Type type, String what, boolean deep, String origInternalNa
           what, binaryClassName
         );
       }
+    } catch (RelatedClassLoadingException e) {
+      // only throw exception if it is not an annotation
+      if (false == isAnnotation) throw e;
     }
     return null;
   }
   
   String checkClassUse(String internalName, String what, String origInternalName) {
-    return checkClassUse(Type.getObjectType(internalName), what, true, origInternalName);
+    return checkClassUse(Type.getObjectType(internalName), what, false, origInternalName);
   }
   
   // TODO: @FunctionalInterface from Java 8 on
@@ -209,7 +213,7 @@ String checkType(Type type) {
       switch (type.getSort()) {
         case Type.OBJECT:
           final String internalName = type.getInternalName();
-          violation = checkClassUse(type, "class/interface", true, internalName);
+          violation = checkClassUse(type, "class/interface", false, internalName);
           if (violation != null) {
             return violation;
           }
@@ -257,8 +261,7 @@ String checkDescriptor(String desc) {
   
   String checkAnnotationDescriptor(Type type, boolean visible) {
     // for annotations, we don't need to look into super-classes, interfaces,...
-    // -> we just check if its disallowed or internal runtime (only if visible)!
-    return checkClassUse(type, "annotation", visible, type.getInternalName());
+    return checkClassUse(type, "annotation", true, type.getInternalName());
   }
   
   void maybeSuppressCurrentGroup(Type annotation) {