Stack Exchange
Appending twice in csv report when trying to list firewall rules in gcp using python
import requests
import json
import re
import sys
import subprocess
import os
from googleapiclient import discovery
from oauth2client.client import GoogleCredentials
from google.oauth2 import service_account


credentials = service_account.Credentials.from_service_account_file("")
service = discovery.build('cloudresourcemanager', 'v1', credentials=credentials)
request = service.projects().list()
token1 = subprocess.Popen("gcloud auth print-access-token", stdout=subprocess.PIPE, shell = True)
token, error = token1.communicate()
token = str(token.decode("utf-8"))
token = token.rstrip("\n")
token = token.rstrip("\r")
Compliance = [""]
ComplianceFlag = 0
PROTOCOL = "-"
PORT = "-"


f = open("xxxxxxxxx.csv", 'w')
f.write("ProjectId, VPC, Rule Name, Direction, Compliance, SourceRange, IPProtocol, Port\n")


while request is not None:
    response = request.execute()
    for project in response.get('projects', []):
        projectid = project['projectId']
        projectname = project['name']


        headers = {
        'Authorization': 'Bearer ' + token,
        'x-goog-user-project': projectid
        }
        count = 0
        try:
            get_url = "https://compute.googleapis.com/compute/v1/projects/"+ projectid +"/global/firewalls"
            get_url_data = requests.get(get_url, headers= headers)
            get_api2_json = json.loads(get_url_data.text)
            for vpc in get_api2_json["items"]:
                vpcname = vpc["network"]
                vpcname = vpcname.split("/")[-1]
                rulename = vpc["name"]
                direction = vpc["direction"]
                try:
                    try:
                        for sr in vpc["sourceRanges"]:
                            if "y.y.y.y/y" in sr:
                                Compliance.append("NonCompliant")
                                ComplianceFlag = 1
                            for allowed in vpc["allowed"]:
                                PROTOCOL=allowed["IPProtocol"]
                                if(PROTOCOL=="all"):
                                    Compliance.append("NonCompliant")
                                    ComplianceFlag = 1
                                try:
                                    for port in allowed["ports"]:
                                        if "22" in port or "139" in port:
                                            Compliance.append("NonCompliantport")
                                            ComplianceFlag = 1
                                        PORT=port
                                        f.write("{},{},{},{},{},{},{},{},{}\n".format(projectid, vpcname, rulename, direction, ' '.join([str(elem) for elem in Compliance]),sr,PROTOCOL,PORT))
                                except KeyError as e:
                                    f.write("{},{},{},{},{},{},{},{},{}\n".format(projectid, vpcname, rulename, direction, ' '.join([str(elem) for elem in Compliance]),sr,PROTOCOL,"-"))
                            if ComplianceFlag == 0:
                                Compliance = [""]
                            ComplianceFlag = 0
                            Compliance = [""]
                    except KeyError as e:
                      
                        f.write("{},{},{},{},{},{},{},{},{},\n".format(projectid, vpcname, rulename, direction, ' '.join([str(elem) for elem in Compliance]),PROTOCOL,PORT))
                        ComplianceFlag = 0
                        Compliance = [""]
                        print("")


                except Exception as e:
                    print(e)
                    pass
        except Exception as e:
            print(e)
            pass
    request = service.projects().list_next(previous_request=request, previous_response=response)
f.close()

print(count)

All i am trying to do here is to generate a csv report to list the firewall rules in gcp and get the compliance check (whether it is complaint or non compliant). When i am trying to append the compliance check value, it is appending twice in the report..
This is where in the code it is appending NonCompliantport Noncompliant port twice....
                                try:
                                    for port in allowed["ports"]:
                                        if "22" in port or "139" in port:
                                            Compliance.append("NonCompliantport")
                                            ComplianceFlag = 1
                                        PORT=port
                                        f.write("{},{},{},{},{},{},{},{},{}\n".format(projectid, vpcname, rulename, direction, ' '.join([str(elem) for elem in Compliance]),sr,PROTOCOL,PORT))
                                except KeyError as e:
                                    f.write("{},{},{},{},{},{},{},{},{}\n".format(projectid, vpcname, rulename, direction, ' '.join([str(elem) for elem in Compliance]),sr,PROTOCOL,"-"))
                            if ComplianceFlag == 0:
                                Compliance = [""]
                            ComplianceFlag = 0
                            Compliance = [""]

Any idea of how to resolve this...

It would be helpful if you included an example input and output so that we can understand what you mean.

I strongly (!) encourage you to use the Compute Engine SDK rather than mash-up subprocess calls and attempting to make the underlying requests manually. As you're learning, it adds complexity and makes it difficult to debug. You are using Cloud Resource Manager SDK to enumerate projects, extending your code to leverage Compute Engine SDK should be trivial.

Can you try to add a check on the complianceFlag like that
                                      for port in allowed["ports"]:
                                        if ("22" in port or "139" in port) and ComplianceFlag == 0:
                                            Compliance.append("NonCompliantport")
                                            ComplianceFlag = 1
                                        PORT=port
                                        f.write("{},{},



Sure let me try that..
