test_class.declarator: ruleMatcher
test_class.modifier: private
test_class.original_string: private RuleMatcher ruleMatcher;
test_class.type: RuleMatcher
test_class.var_name: ruleMatcher
test_class.file: checker/src/test/java/com/fooock/robotstxt/api/engine/RuleMatcherTest.java
test_class.identifier: RuleMatcherTest
test_case.body: @Test
      public void testAllWildcardPattern() throws Exception {
          InputStream stream = this.getClass().getClassLoader().getResourceAsStream("wildcard-disallow-all.json");
          String content = TestUtils.readFile(stream);
  
          assertFalse(ruleMatcher.canCrawl("*", "/hello?user=javi", content));
          assertFalse(ruleMatcher.canCrawl("*", "/disallow/this/rule?user=javi", content));
  
          assertTrue(ruleMatcher.canCrawl("007", "/test?user=javi", content));
          assertTrue(ruleMatcher.canCrawl("007", "/test/allow/this/hello", content));
      }
test_case.class_method_signature: RuleMatcherTest.testAllWildcardPattern()
test_case.full_signature: @Test public void testAllWildcardPattern()
test_case.identifier: testAllWildcardPattern
test_case.invocations: getResourceAsStream getClassLoader getClass readFile assertFalse canCrawl assertFalse canCrawl assertTrue canCrawl assertTrue canCrawl
test_case.modifiers: @Test public
test_case.parameters: ()
test_case.return: void
test_case.signature: void testAllWildcardPattern()
focal_class.declarator: ALLOW = "allow"
focal_class.modifier: private static final
focal_class.original_string: private static final String ALLOW = "allow";
focal_class.type: String
focal_class.var_name: ALLOW
focal_class.declarator: DISALLOW = "disallow"
focal_class.modifier: private static final
focal_class.original_string: private static final String DISALLOW = "disallow";
focal_class.type: String
focal_class.var_name: DISALLOW
focal_class.declarator: ALL = "*"
focal_class.modifier: private static final
focal_class.original_string: private static final String ALL = "*";
focal_class.type: String
focal_class.var_name: ALL
focal_class.declarator: DOLLAR = "$"
focal_class.modifier: private static final
focal_class.original_string: private static final String DOLLAR = "$";
focal_class.type: String
focal_class.var_name: DOLLAR
focal_class.declarator: gson
focal_class.modifier: private final
focal_class.original_string: private final Gson gson;
focal_class.type: Gson
focal_class.var_name: gson
focal_class.declarator: type = new TypeToken<Content>() {
      }.getType()
focal_class.modifier: private final
focal_class.original_string: private final Type type = new TypeToken<Content>() {
      }.getType();
focal_class.type: Type
focal_class.var_name: type
focal_class.file: checker/src/main/java/com/fooock/robotstxt/api/engine/RuleMatcher.java
focal_class.identifier: RuleMatcher
focal_class.class_method_signature: RuleMatcher.canCrawl(String agent, String path, String rules)
focal_class.full_signature: public boolean canCrawl(String agent, String path, String rules)
focal_class.identifier: canCrawl
focal_class.modifiers: public
focal_class.parameters: (String agent, String path, String rules)
focal_class.return: boolean
focal_class.signature: boolean canCrawl(String agent, String path, String rules)
focal_class.class_method_signature: RuleMatcher.noneMatch(String agent, List<String> userAgents)
focal_class.full_signature: private boolean noneMatch(String agent, List<String> userAgents)
focal_class.identifier: noneMatch
focal_class.modifiers: private
focal_class.parameters: (String agent, List<String> userAgents)
focal_class.return: boolean
focal_class.signature: boolean noneMatch(String agent, List<String> userAgents)
focal_class.class_method_signature: RuleMatcher.checkGroupRules(Group group, String path)
focal_class.full_signature: private boolean checkGroupRules(Group group, String path)
focal_class.identifier: checkGroupRules
focal_class.modifiers: private
focal_class.parameters: (Group group, String path)
focal_class.return: boolean
focal_class.signature: boolean checkGroupRules(Group group, String path)
focal_class.class_method_signature: RuleMatcher.matchesRule(String path, String directive, Rule rule)
focal_class.full_signature: private boolean matchesRule(String path, String directive, Rule rule)
focal_class.identifier: matchesRule
focal_class.modifiers: private
focal_class.parameters: (String path, String directive, Rule rule)
focal_class.return: boolean
focal_class.signature: boolean matchesRule(String path, String directive, Rule rule)
focal_class.class_method_signature: RuleMatcher.checkRulePath(String path, String directive, Rule rule)
focal_class.full_signature: private boolean checkRulePath(String path, String directive, Rule rule)
focal_class.identifier: checkRulePath
focal_class.modifiers: private
focal_class.parameters: (String path, String directive, Rule rule)
focal_class.return: boolean
focal_class.signature: boolean checkRulePath(String path, String directive, Rule rule)
focal_method.body: public boolean canCrawl(String agent, String path, String rules) {
          // Fixes #29
          if (path == null || path.isEmpty()) path = "/";
  
          // if path is /robots.txt then allow always
          if ("/robots.txt".equals(path)) return true;
  
          Content content = gson.fromJson(rules, type);
          List<Group> groups = content.getGroups();
  
          // if no groups found, then allow all
          if (groups.isEmpty()) return true;
  
          List<String> userAgents = groups.stream()
                  .flatMap(group -> group.getUserAgents().stream().map(String::toLowerCase))
                  .collect(Collectors.toList());
  
          if (userAgents.isEmpty()) return true;
          if (userAgents.parallelStream().noneMatch(ALL::equals) && ALL.equals(agent)) return true;
  
          agent = agent.toLowerCase();
          if (noneMatch(agent, userAgents) && noneMatch(ALL, userAgents) && !ALL.equals(agent)) return true;
  
          // Flag used to check if the given user agent match any saved agent.
          // Note that if no user agent match, it will use * as base user agent
          boolean foundUserAgent = false;
  
          for (Group group : groups) {
              // Directive priority is decided using length, that is, longest first
              Collections.sort(group.getRules());
  
              for (String ua : group.getUserAgents()) {
                  // Check if user agent match. User agent case is ignored as
                  // google documentation says
                  if (agent.contains(ua.toLowerCase())) {
                      foundUserAgent = true;
                      // If rules check, then allow
                      if (checkGroupRules(group, path)) return true;
                  }
              }
  
              // Check with default user agent (*)
              if (!foundUserAgent) {
                  for (String ua : group.getUserAgents()) {
                      // Continue loop until * found
                      if (!ALL.equals(ua)) continue;
                      if (checkGroupRules(group, path)) return true;
                  }
              }
          }
          return false;
      }
focal_method.class_method_signature: RuleMatcher.canCrawl(String agent, String path, String rules)
focal_method.full_signature: public boolean canCrawl(String agent, String path, String rules)
focal_method.identifier: canCrawl
focal_method.invocations: isEmpty equals fromJson getGroups isEmpty collect flatMap stream map stream getUserAgents toList isEmpty noneMatch parallelStream equals toLowerCase noneMatch noneMatch equals sort getRules getUserAgents contains toLowerCase checkGroupRules getUserAgents equals checkGroupRules
focal_method.modifiers: public
focal_method.parameters: (String agent, String path, String rules)
focal_method.return: boolean
focal_method.signature: boolean canCrawl(String agent, String path, String rules)