import numpy as np
import warnings

# Card names

# Base game cards
cellar = "Kjeller"
chapel = "Kapell"
moat = "Vollgrav"
chancellor = "Rikskanseller"
village = "Landsby"
woodcutter = "Tømmerhogger"
workshop = "Verksted"
bureaucrat = "Byråkrat"
feast = "Gilde"
gardens = "Park"
militia = "Hird"
moneylender = "Pengeutlåner"
remodel = "Ombygging"
smithy = "Smie"
spy = "Spion"
thief = "Tyv"
throne_room = "Tronsal"
council_room = "Rådssal"
festival = "Festival"
laboratory = "Laboratorium"
library = "Bibliiotek"
market = "Marked"
mine = "Gruve"
witch = "Heks"
adventurer = "Eventyrer"

# Seaside cards
ambassador = "Ambassador"
bazaar = "Bazaar"
caravan = "Caravan"
cutpurse = "Cutpurse"
embargo = "Embargo"
explorer = "Explorer"
fishing_village = "Fishing Village"
ghost_ship = "Ghost Ship"
haven = "Haven"
island = "Island"
lighthouse = "Lighthouse"
lookout = "Lookout"
merchant_ship = "Merchant Ship"
native_village = "Native Village"
navigator = "Navigator"
outpost = "Outpost"
pearl_diver = "Pearl Diver"
pirate_ship = "Pirate Ship"
salvager = "Salvager"
sea_hag = "Sea Hag"
smugglers = "Smugglers"
tactician = "Tactician"
treasure_map = "Treasure Map"
treasury = "Treasury"
warehouse = "Warehouse"
wharf = "Wharf"

# Prosperity cards
workers_village = "Arbeiderlandsby"
bank = "Bank"
bishop = "Biskop"
counting_house = "Bokholderi"
venture = "Forretninger"
peddler = "Gateselger"
trade_route = "Handelsrute"
city = "Kjøpstad"
royal_seal = "Kongelig Segl"
kings_court = "Kongens Hoff"
contraband = "Kontrabande"
goons = "Ludendreiere"
loan = "Lån"
rabble = "Mobb"
monument = "Monument"
mint = "Myntmakeri"
expand = "Påbygning"
mountebank = "Sjarlatan"
hoard = "Skattkammer"
quarry = "Steinbrudd"
grand_market = "Stormarked"
forge = "Sverdmakeri"
talisman = "Talisman"
watchtower = "Vakttårn"
vault = "Velv"

# Alchemy cards
transmute = "Transmute"
vineyard = "Vineyard"
herbalist = "Herbalist"
apothecary = "Apothecary"
scrying_pool = "Scrying Pool"
university = "University"
alchemist = "Alchemist"
familiar = "Familiar"
philosophers_stone = "Philosopher\'s Stone"
golem = "Golem"
apprentice = "Apprentice"
possession = "Possession"

# Dark Ages cards
altar = "Altar"
armory = "Armory"
band_of_misfits = "Band of Misfits"
bandit_camp = "Bandit Camp"
beggar = "Beggar"
catacombs = "Catacombs"
count = "Count"
counterfeit = "Counterfeit"
cultist = "Cultist"
death_cart = "Death Cart"
forager = "Forager"
fortress = "Fortress"
graverobber = "Graverobber"
hunting_grounds = "Hunting Grounds"
ironmonger = "Ironmonger"
junk_dealer = "Junk Dealer"
marauder = "Marauder"
market_square = "Market Square"
mystic = "Mystic"
pillage = "Pillage"
poor_house = "Poor House"
procession = "Procession"
rebuild = "Rebuild"
rogue = "Rogue"
sage = "Sage"
scavenger = "Scavenger"
squire = "Squire"
hermit = "Hermit + Madman"
urchin = "Urchin + Mercenary"
rats = "Rats"
storeroom = "Storeroom"
vagrant = "Vagrant"
wandering_minstrel = "Wandering Minstrel"
feodum = "Feodum"
knights = "Knights"

# Empires cards
archive = "Archive"
capital = "Capital"
catapult_rocks = "Catapult + Rocks"
chariot_race = "Chariot Race"
charm = "Charm"
city_quarter = "City Quarter"
crown = "Crown"
encampment_plunder = "Encampment + Plunder"
enchantress = "Enchantress"
engineer = "Engineer"
farmers_market = "Farmers\' Market"
forum = "Forum"
gladiator_fortune = "Gladiator + Fortune"
groundskeeper = "Groundskeeper"
legionary = "Legionary"
overlord = "Overlord"
patrician_emporium = "Patrician + Emporium"
royal_blacksmith = "Royal Blacksmith"
sacrifice = "Sacrifice"
settlers_bustling_village = "Settlers + Bustling Village"
temple = "Temple"
villa = "Villa"
wild_hunt = "Wild Hunt"
castles = "Castles"

# Renaissance cards
border_guard = "Border Guard"
ducat = "Ducat"
lackeys = "Lackeys"
acting_troupe = "Acting Troupe"
cargo_ship = "Cargo Ship"
experiment = "Experiment"
improve = "Improve"
flag_bearer = "Flag Bearer"
hideout = "Hideout"
inventor = "Inventor"
mountain_village = "Mountain Village"
patron = "Patron"
priest = "Priest"
research = "Research"
silk_merchant = "Silk Merchant"
old_witch = "Old Witch"
recruiter = "Recruiter"
scepter = "Scepter"
scholar = "Scholar"
sculptor = "Sculptor"
seer = "Seer"
spices = "Spices"
swashbuckler = "Swashbuckler"
treasurer = "Treasurer"
villain = "Villain"

# Allies cards
augurs = "Augurs"
barbarian = "Barbarian"
bauble = "Bauble"
broker = "Broker"
capital_city = "Capital City"
carpenter = "Carpenter"
clashes = "Clashes"
contract = "Contract"
courier = "Courier"
emissary = "Emissary"
forts = "Forts"
galleria = "Galleria"
guildmaster = "Guildmaster"
highwayman = "Highwayman"
hunter = "Hunter"
importer = "Importer"
innkeeper = "Innkeeper"
marquis = "Marquis"
merchant_camp = "Merchant Camp"
modify = "Modify"
odysseys = "Odysseys"
royal_gallery = "Royal Gallery"
sentinel = "Sentinel"
skirmisher = "Skirmisher"
specialist = "Specialist"
swap = "Swap"
sycophant = "Sycophant"
town = "Town"
townsfolk = "Townsfolk"
underling = "Underling"
wizards = "Wizards"

# Projects
cathedral = "Cathedral"
city_gate = "City Gate"
pageant = "Pageant"
sewers = "Sewers"
star_chart = "Star Chart"
exploration = "Exploration"
fair = "Fair"
silos = "Silos"
sinister_plot = "Sinister Plot"
academy = "Academy"
capitalism = "Capitalism"
fleet = "Fleet"
guildhall = "Guildhall"
piazza = "Piazza"
road_network = "Road Network"
barracks = "Barracks"
crop_rotation = "Crop Rotation"
innovation = "Innovation"
canal = "Canal"
citadel = "Citadel"

# Events
triumph = "Triumph"
annex = "Annex"
donate = "Donate"
advance = "Advance"
delve = "Delve"
tax = "Tax"
banquet = "Banquet"
ritual = "Ritual"
salt_the_earth = "Salt the Earth"
wedding = "Wedding"
windfall = "Windfall"
conquest = "Conquest"
dominate = "Dominate"

# Landmarks
aqueduct = "Aqueduct"
arena = "Arena"
bandit_fort = "Bandit Fort"
basilica = "Basilica"
baths = "Baths"
battlefield = "Battlefield"
colonnade = "Colonnade"
defiled_shrine = "Defiled Shrine"
fountain = "Fountain"
keep = "Keep"
labirynth = "Labirynth"
mountain_pass = "Mountain Pass"
museum = "Museum"
obelisk = "Obelisk"
orchard = "Orchard"
palace = "Palace"
tomb = "Tomb"
tower = "Tower"
triumphal_arch = "Triumphal Arch"
wall = "Wall"
wolf_den = "Wolf Den"

# Allies
architects_guild = "Architects\' Guild"
band_of_nomads = "Band of Nomads"
cave_dwellers = "Cave Dwellers"
circle_of_witches = "Circle of Witches"
city_state = "City-state"
coastal_haven = "Coastal Haven"
crafters_guild = "Crafters\' Guild"
desert_guides = "Desert Guides"
family_of_inventors = "Family of Inventors"
fellowship_of_scribes = "Fellowship of Scribes"
forest_dwellers = "Forest Dwellers"
gang_of_pickpockets = "Gang of Pickpockets"
island_folk = "Island Folk"
league_of_bankers = "League of Bankers"
league_of_shopkeepers = "League of Shopkeepers"
market_towns = "Market Towns"
mountain_folk = "Mountain Folk"
order_of_astrologers = "Order of Astrologers"
order_of_masons = "Order of Masons"
peaceful_cult = "Peaceful Cult"
plateau_shepherds = "Plateau Shepherds"
trappers_lodge = "Trappers\' Lodge"
woodworkers_guild = "Woodworkers\' Guild"

liasion_cards = [bauble,
                 sycophant,
                 importer,
                 wizards,
                 underling,
                 broker,
                 contract,
                 emissary,
                 guildmaster]

base = [moat,
        library,
        bureaucrat,
        adventurer,
        festival,
        feast,
        mine,
        witch,
        militia,
        chapel,
        cellar,
        laboratory,
        village,
        market,
        remodel,
        moneylender,
        chancellor,
        council_room,
        smithy,
        spy,
        throne_room,
        thief,
        woodcutter,
        workshop,
        gardens]

seaside = [ambassador,
           bazaar,
           caravan,
           cutpurse,
           embargo,
           explorer,
           fishing_village,
           ghost_ship,
           haven,
           island,
           lighthouse,
           lookout,
           merchant_ship,
           native_village,
           navigator,
           outpost,
           pearl_diver,
           pirate_ship,
           salvager,
           sea_hag,
           smugglers,
           tactician,
           treasure_map,
           treasury,
           warehouse,
           wharf]

alchemy = [transmute,
           vineyard,
           herbalist,
           apothecary,
           scrying_pool,
           university,
           alchemist,
           familiar,
           philosophers_stone,
           golem,
           apprentice,
           possession]

prosperity = [workers_village,
              bank,
              bishop,
              counting_house,
              venture,
              peddler,
              trade_route,
              city,
              royal_seal,
              kings_court,
              contraband,
              goons,
              loan,
              rabble,
              monument,
              mint,
              expand,
              mountebank,
              hoard,
              quarry,
              grand_market,
              forge,
              talisman,
              watchtower,
              vault]

# cornucopia

# hinterlands

dark_ages = [altar,
             armory,
             band_of_misfits,
             bandit_camp,
             beggar,
             catacombs,
             count,
             counterfeit,
             cultist,
             death_cart,
             forager,
             fortress,
             graverobber,
             hunting_grounds,
             ironmonger,
             junk_dealer,
             marauder,
             market_square,
             mystic,
             pillage,
             poor_house,
             procession,
             rebuild,
             rogue,
             sage,
             scavenger,
             squire,
             hermit,
             urchin,
             rats,
             storeroom,
             vagrant,
             wandering_minstrel,
             feodum,
             knights]

# guilds

# adventures

empires = [archive,
           capital,
           catapult_rocks,
           chariot_race,
           charm,
           city_quarter,
           crown,
           encampment_plunder,
           enchantress,
           engineer,
           farmers_market,
           forum,
           gladiator_fortune,
           groundskeeper,
           legionary,
           overlord,
           patrician_emporium,
           royal_blacksmith,
           sacrifice,
           settlers_bustling_village,
           temple,
           villa,
           wild_hunt,
           castles]

# nocturne

renaissance = [border_guard,
               ducat,
               lackeys,
               acting_troupe,
               cargo_ship,
               experiment,
               improve,
               flag_bearer,
               hideout,
               inventor,
               mountain_village,
               patron,
               priest,
               research,
               silk_merchant,
               old_witch,
               recruiter,
               scepter,
               scholar,
               sculptor,
               seer,
               spices,
               swashbuckler,
               treasurer,
               villain]

# menagerie

allies = [augurs,
          barbarian,
          bauble,
          broker,
          capital_city,
          carpenter,
          clashes,
          contract,
          courier,
          emissary,
          forts,
          galleria,
          guildmaster,
          highwayman,
          hunter,
          importer,
          innkeeper,
          marquis,
          merchant_camp,
          modify,
          odysseys,
          royal_gallery,
          sentinel,
          skirmisher,
          specialist,
          swap,
          sycophant,
          town,
          townsfolk,
          underling,
          wizards]

projects = [cathedral,
            city_gate,
            pageant,
            sewers,
            star_chart,
            exploration,
            fair,
            silos,
            sinister_plot,
            academy,
            capitalism,
            fleet,
            guildhall,
            piazza,
            road_network,
            barracks,
            crop_rotation,
            innovation,
            canal,
            citadel]

events = [triumph,
          annex,
          donate,
          advance,
          delve,
          tax,
          banquet,
          ritual,
          salt_the_earth,
          wedding,
          windfall,
          conquest,
          dominate]

landmarks = [aqueduct,
             arena,
             bandit_fort,
             basilica,
             baths,
             battlefield,
             colonnade,
             defiled_shrine,
             fountain,
             keep,
             labirynth,
             mountain_pass,
             museum,
             obelisk,
             orchard,
             palace,
             tomb,
             tower,
             triumphal_arch,
             wall,
             wolf_den]

ally_cards = [architects_guild,
              band_of_nomads,
              cave_dwellers,
              circle_of_witches,
              city_state,
              coastal_haven,
              crafters_guild,
              desert_guides,
              family_of_inventors,
              fellowship_of_scribes,
              forest_dwellers,
              gang_of_pickpockets,
              island_folk,
              league_of_bankers,
              league_of_shopkeepers,
              market_towns,
              mountain_folk,
              order_of_astrologers,
              order_of_masons,
              peaceful_cult,
              plateau_shepherds,
              trappers_lodge,
              woodworkers_guild]

# Card classes
# actioner = "Actioner"       # Give actions or play action cards
# drawer = "Drawer"           # Draw cards
# buyer = "Buyer"             # Grant buys
# moneymaker = "Moneymaker"   # Grant coins
# trasher = "Trasher"         # Possibility to trash cards
# attacker = "Attacker"       # Attack card
# reactioner = "Reactioner"   # Reaction card
# treasurer = "Treasurer"     # Treasure card
# alt_VP = "Alt_VP"           # Victory card
# duration = "Duration"       # Duration card
# other = "Other"             # Other cards

# card_class_dict = {
#     actioner: [village,
#                festival,
#                throne_room,
#                native_village,
#                fishing_village,
#                bazaar,
#                workers_village,
#                city,
#                kings_court,
#                university,
#                golem,
#                squire,
#                fortress,
#                procession,
#                wandering_minstrel,
#                bandit_camp]
# }

even_spread = "even spread"
weighted = "weighted"
random = "random"

type = random
sets = [base, seaside, prosperity, alchemy, dark_ages, empires, renaissance, allies]
landscape_cards = projects + events + landmarks

# weights must sum to 10
# weights = [1, 2, 2, 0, 0, 5]          # Renaissance
# weights = [1, 1, 1, 5, 1, 1]          # Alchemy
# weights = [2, 2, 2, 0, 2, 2]          # Not Alchemy
# weights = [4, 3, 3, 0, 0, 0]          # Simple
# weights = [5, 0, 5, 0, 0, 0]          # Base + Prosperity
weights = [0, 1, 1, 0, 1, 3, 1, 3]      # Empires + Allies
include = []
exclude = [possession, barbarian]

cards = []

if type == even_spread:
    np.random.shuffle(sets)
    for s in sets:
        np.random.shuffle(s)
    i = 0
    while len(cards) < 10 - len(include):
        c = sets[i % len(sets)][i // len(sets)]
        if c in exclude:
            sets[i % len(sets)].remove(c)
            continue
        cards.append(c)
        i += 1

elif type == weighted:
    for s in sets:
        np.random.shuffle(s)
    for i in range(len(weights)):
        weights[i] /= 10
    # print(f"weights: {weights}")
    while len(cards) < 10 - len(include):
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            set_choice = np.random.choice(sets, p=weights)
        if not set_choice[0] in exclude:
            cards.append(set_choice[0])
        set_choice.pop(0)

else:
    for s in sets:
        cards += s
    np.random.shuffle(cards)
    for card in exclude:
        cards.remove(card)

selected_cards = include + cards[:10 - len(include)]

print(f"Cards: {selected_cards}")

if landscape_cards:
    num_side_cards = np.random.randint(1, 3)
    np.random.shuffle(landscape_cards)
    print(f"Extra cards: {landscape_cards[:num_side_cards]}")

if len(set(selected_cards).intersection(liasion_cards)) > 0:
    np.random.shuffle(ally_cards)
    print(f"Ally: {ally_cards[0]}")


# User defined data
directories_to_ignore = { "iTunes" }

# Rest

import os
import subprocess
import itertools
import re
import math
from collections import namedtuple
OggInfo = namedtuple("OggInfo", ["title", "artist", "album", "tracknumber", "length"])

from mutagen.oggvorbis import OggVorbis

def run_ogg_info(filename):
    """Uses `Mutagen` library instead"""
    # This will capture, as a dictionary, the title, album etc.
    # However, each member will be a list, so grab 1st entry.
    data = OggVorbis(filename)
    # Can extract the length from data.info member
    length = math.ceil(data.info.length)
    # Return
    return OggInfo(data["title"][0], data["artist"][0], data["album"][0],
                   data["tracknumber"][0], length)

def get_sorted_filenames(directory):
    walker = os.walk(directory)
    _, dirs, files = next(walker)
    if len(dirs) > 0:
        raise Exception("{} contains subdirectories, which we don't expect!".format(directory))
    files.sort()
    return files

def make_m3u(directory, output_filename):
    # This is a slight hack...  We assemble the output in a single string
    # and then write to a file in one go.  This means the chance of a Ctrl-C
    # interrupt leaving a half-formed file is much smaller.
    m3u = ""
    m3u += "#EXTM3U\n"
    for file in get_sorted_filenames(directory):
        data = run_ogg_info(os.path.join(directory, file))
        if data.length == None or data.artist == None or data.title == None:
            raise Exception("{} doesn't contain enough data: {}".format(file, data))
        m3u += "#EXTINF:{},{} - {}\n".format(data.length, data.artist, data.title)
        fullfilename = os.path.join(directory, file)
        base, filename = os.path.split(fullfilename)
        _, firstdir = os.path.split(base)
        m3u += os.path.join(firstdir, filename) + "\n"
    # Now write to file
    with open(output_filename, "w") as f:
        print(m3u, file=f, end="")

# Main script

# Parse arguments
import argparse

parser = argparse.ArgumentParser(description="Generate m3u files.")
parser.add_argument("task", default="new", choices=["new", "all"],
    help = "new: Generate only new m3u files; all: Generate all files.")
parser.add_argument("--ignore", nargs="*",
    help = "List of directories to ignore.")
args = parser.parse_args()

if args.ignore:
    for d in args.ignore:
        directories_to_ignore.add(d)

# Walk current directory

walker = os.walk(".")
root, dirs, files = next(walker)
for directory in dirs:
    if directory in directories_to_ignore:
        continue
    filename = directory+".m3u"
    if args.task == "new" and filename in files:
        continue
    print(directory)
    make_m3u(directory, filename)


import os
import logging
import isceobj
import stdproc

from iscesys.ImageUtil.ImageUtil import ImageUtil as IU

logger = logging.getLogger('isce.isceProc.runResamp_image')

def runResamp_image(self):
    refPol = self._isce.refPol
    stdWriter = self._stdWriter
    dopplerCentroid = self._isce.dopplerCentroid
    looks = self._isce.numberLooks
    numFitCoeff = self._isce.numberFitCoefficients
    offsetImageName = self._isce.offsetImageName
    pixelSpacing = self._isce.slantRangePixelSpacing
    lines = self._isce.numberResampLines
    for sceneid1, sceneid2 in self._isce.pairsToCoreg:
        pair = (sceneid1, sceneid2)
        imageSlc1 = self._isce.slcImages[sceneid1][refPol]
        frame1 = self._isce.frames[sceneid1][refPol]
        instrument = frame1.getInstrument()
        offsetField = self._isce.refinedOffsetFields[pair] #offsetField is the same for all pols
        imageSlc2 = self._isce.slcImages[sceneid2][refPol]
        catalog = isceobj.Catalog.createCatalog(self._isce.procDoc.name)
        sid = self._isce.formatname(pair)
        offsetFilename = os.path.join(self.getoutputdir(sceneid1, sceneid2), self._isce.formatname(pair, ext=offsetImageName))
        imageAz, imageRn = run(imageSlc1, imageSlc2, offsetField, instrument, dopplerCentroid, looks, lines, numFitCoeff, pixelSpacing, offsetFilename, stdWriter, catalog=catalog, sceneid=sid)
        self._isce.offsetAzimuthImages[pair] = imageAz
        self._isce.offsetRangeImages[pair] = imageRn

def run(imageSlc1, imageSlc2, offsetField, instrument, dopplerCentroid, looks, lines, numFitCoeff, pixelSpacing, offsetFilename, stdWriter, catalog=None, sceneid='NO_ID'):
    widthSlc = max(imageSlc1.getWidth(), imageSlc2.getWidth())
    dopplerCoeff = dopplerCentroid.getDopplerCoefficients(inHz=False)

    path, filename = os.path.split(offsetFilename)
    offsetAz = os.path.join(path, 'azimuth_' + filename)
    offsetRn = os.path.join(path, 'range_' + filename)
    widthOffset = int(widthSlc / looks)
    imageAz = isceobj.createOffsetImage()
    imageAz.setFilename(offsetAz)
    imageAz.setWidth(widthOffset)
    imageRn = isceobj.createOffsetImage()
    imageRn.setFilename(offsetRn)
    imageRn.setWidth(widthOffset)

    objAz = isceobj.createOffsetImage()
    objRn = isceobj.createOffsetImage()
    IU.copyAttributes(imageAz, objAz)
    IU.copyAttributes(imageRn, objRn)
    objAz.setAccessMode('write')
    objAz.createImage()
    objRn.setAccessMode('write')
    objRn.createImage()

    objResamp_image = stdproc.createResamp_image()
    objResamp_image.wireInputPort(name='offsets', object=offsetField)
    objResamp_image.wireInputPort(name='instrument', object=instrument)
    objResamp_image.setSlantRangePixelSpacing(pixelSpacing)
    objResamp_image.setDopplerCentroidCoefficients(dopplerCoeff)
    objResamp_image.setNumberLooks(looks)
    objResamp_image.setNumberLines(lines)
    objResamp_image.setNumberRangeBin(widthSlc)
    objResamp_image.setNumberFitCoefficients(numFitCoeff)
    #set the tag used in the outfile. each message is precided by this tag
    #is the writer is not of "file" type the call has no effect
    objResamp_image.stdWriter = stdWriter.set_file_tags("resamp_image",
                                                        "log",
                                                        "err",
                                                        "out")

    objResamp_image.resamp_image(objRn, objAz)

    if catalog is not None:
        # Record the inputs and outputs
        isceobj.Catalog.recordInputsAndOutputs(catalog, objResamp_image,
                                               "runResamp_image.%s" % sceneid,
                                               logger,
                                               "runResamp_image.%s" % sceneid)

    objRn.finalizeImage()
    objAz.finalizeImage()

    return imageAz, imageRn


#!/usr/bin/env python3

"""
Script to make matplotlib plots from TRExFitter output.
"""

# fmt: off
import matplotlib
matplotlib.use("pdf")
import matplotlib.font_manager as font_manager
#import os
#if os.environ.get("HELVETICA_MPL"):
#    curdir = os.path.dirname(os.path.abspath(__file__))
#    print(curdir)
#    fontprop_reg = font_manager.FontProperties(fname="{}/Helvetica/Regular.ttf".format(curdir))
#    fontprop_atl = font_manager.FontProperties(fname="{}/Helvetica/Bold_Italic.ttf".format(curdir))
#    matplotlib.rcParams["font.family"] = fontprop_reg.get_name()
#    print(fontprop_reg.get_name())
#matplotlib.rcParams['pdf.fonttype'] = 42
#matplotlib.rcParams['ps.fonttype'] = 42
#matplotlib.rcParams['text.usetex'] = True
matplotlib.rcParams["axes.labelsize"] = 14
matplotlib.rcParams["font.size"] = 12
matplotlib.rcParams["xtick.top"] = True
matplotlib.rcParams["ytick.right"] = True
matplotlib.rcParams["xtick.direction"] = "in"
matplotlib.rcParams["ytick.direction"] = "in"
matplotlib.rcParams["xtick.labelsize"] = 12
matplotlib.rcParams["ytick.labelsize"] = 12
matplotlib.rcParams["xtick.minor.visible"] = True
matplotlib.rcParams["ytick.minor.visible"] = True
matplotlib.rcParams["xtick.major.width"] = 0.8
matplotlib.rcParams["xtick.minor.width"] = 0.8
matplotlib.rcParams["xtick.major.size"] = 7.0
matplotlib.rcParams["xtick.minor.size"] = 4.0
matplotlib.rcParams["xtick.major.pad"] = 1.5
matplotlib.rcParams["xtick.minor.pad"] = 1.4
matplotlib.rcParams["ytick.major.width"] = 0.8
matplotlib.rcParams["ytick.minor.width"] = 0.8
matplotlib.rcParams["ytick.major.size"] = 7.0
matplotlib.rcParams["ytick.minor.size"] = 4.0
matplotlib.rcParams["ytick.major.pad"] = 1.5
matplotlib.rcParams["ytick.minor.pad"] = 1.4
matplotlib.rcParams["legend.frameon"] = False
matplotlib.rcParams["legend.numpoints"] = 1
matplotlib.rcParams["legend.fontsize"] = 11
matplotlib.rcParams["legend.handlelength"] = 1.5
matplotlib.rcParams["axes.formatter.limits"] = [-4, 4]
matplotlib.rcParams["axes.formatter.use_mathtext"] = True
# fmt: on

import argparse
from rexart.pulls import run_pulls
from rexart.stacks import run_stacks

import logging

def get_args():
    # fmt: off
    parser = argparse.ArgumentParser()
    subcommands = parser.add_subparsers(dest="action", help="main action")

    common_parser = argparse.ArgumentParser(add_help=False)
    common_parser.add_argument("--shrink", action="store_true", help="shrink output PDFs via ghostscript")
    common_parser.add_argument("--debug", action="store_true", help="set logging level to debug")

    stacks = subcommands.add_parser("stacks", help="Generate stack plots", parents=[common_parser])
    stacks.add_argument("workspace", type=str, help="TRExFitter workspace")
    stacks.add_argument("-o", "--out-dir", type=str, help="output directory for plots")
    stacks.add_argument("--lumi", type=str, default="139", help="Integrated lumi. for text")
    stacks.add_argument("--do-postfit", action="store_true", help="produce post fit plots as well")
    stacks.add_argument("--skip-regions", type=str, default=None, help="skip regions based on regex")
    stacks.add_argument("--band-style", type=str, choices=["hatch", "shade"], default="hatch", help="band art")
    stacks.add_argument("--legend-ncol", type=int, choices=[1, 2], default=1, help="number of legend columns")
    stacks.set_defaults(func=run_stacks)

    pulls = subcommands.add_parser("pulls", help="pull plots", parents=[common_parser])
    pulls.add_argument("workspace", type=str, help="TRExFitter workspace")
    pulls.add_argument("config", type=str, help="TRExFitter config")
    pulls.add_argument("-o", "--out-dir", type=str, help="output directory")
    pulls.add_argument("--no-text", action="store_true", help="don't print values on plots")
    pulls.set_defaults(func=run_pulls)
    # fmt: on

    return (parser.parse_args(), parser)

def cli():
    args, parser = get_args()
    if args.action is None:
        parser.print_help()
        return 0
    default_level = logging.INFO
    if args.debug:
        default_level = logging.DEBUG
        logging.getLogger("matplotlib.backends.backend_pdf").setLevel(logging.INFO)
        logging.getLogger("matplotlib.font_manager").setLevel(logging.INFO)
    # fmt: off
    logging.basicConfig(level=default_level, format="{:20}  %(levelname)s  %(message)s".format("[%(name)s]"))
    logging.addLevelName(logging.WARNING, "\033[1;31m{:8}\033[1;0m".format(logging.getLevelName(logging.WARNING)))
    logging.addLevelName(logging.ERROR, "\033[1;35m{:8}\033[1;0m".format(logging.getLevelName(logging.ERROR)))
    logging.addLevelName(logging.INFO, "\033[1;32m{:8}\033[1;0m".format(logging.getLevelName(logging.INFO)))
    logging.addLevelName(logging.DEBUG, "\033[1;34m{:8}\033[1;0m".format(logging.getLevelName(logging.DEBUG)))
    # fmt: on
    args.func(args)
    return 0


#User function Template for python3

class Solution:
    def mostBalloons(self, N, arr):
        # Code here
        ans=1
        for i in range(N):
            count=dict()
            samepos=0
            for j in range(N):
                xdiff=arr[j][0]-arr[i][0]
                ydiff=arr[j][1]-arr[i][1]
                if xdiff==0 and ydiff==0:
                    samepos+=1
                else:
                    slope=float("inf") if ydiff==0 else xdiff/ydiff
                    count[slope]=count.get(slope,0)+1
            ans=max(ans,max(count.values())+samepos)
        return ans
#{
#  Driver Code Starts
#Initial Template for Python 3

import math
if __name__ == '__main__':

    T = int(input())
    while T > 0:
        N = int(input())
        arr=[[] for j in range(N)]
        for j in range(2):
            inp = [int(i) for i in input().split()]
            for i in range(N):
                arr[i].append(inp[i])
        ob = Solution()
        print(ob.mostBalloons(N, arr))

        T -= 1
# } Driver Code Ends

Hit most Balloons - GFG/hit-most-balloons.py


from django.shortcuts import render
from django.views.generic import View
from pure_pagination import Paginator, EmptyPage, PageNotAnInteger
from django.http import JsonResponse, HttpResponse
from django.db.models import Q

from .models import *
from .forms import UserAskForm
from operation.models import UserFavorite
from courses.models import Course
from utils.mixin_utils import LoginRequiredMixin

# Create your views here.

class OrgView(View):
    '''
    课程机构列表功能
    '''

    def get(self, request):
        # 课程机构
        all_orgs = CourseOrg.objects.all()
        hot_orgs = all_orgs.order_by('-click_nums')[:3]
        # 城市
        all_citys = CityDict.objects.all()
        # 城市筛选
        city_id = request.GET.get('city', '')
        if city_id:
            all_orgs = all_orgs.filter(city_id=int(city_id))
        # 类别筛选
        category = request.GET.get('ct', '')
        if category:
            all_orgs = all_orgs.filter(category=category)
        # 搜索
        search_keywords = request.GET.get('keywords', '')
        if search_keywords:
            all_orgs = all_orgs.filter(
                Q(name__icontains=search_keywords) | Q(desc__icontains=search_keywords))

        # 排序
        sort = request.GET.get('sort', '')
        if sort:
            if sort == 'students':
                all_orgs = all_orgs.order_by('-students')
            elif sort == 'courses':
                all_orgs = all_orgs.order_by('-course_nums')

        org_nums = all_orgs.count()
        try:
            page = request.GET.get('page', 1)
        except PageNotAnInteger:
            page = 1
        p = Paginator(all_orgs, 2, request=request)
        orgs = p.page(page)
        return render(request, 'org-list.html', locals())

class AddUserAskView(View):

    def post(self, request):
        userask_form = UserAskForm(request.POST)
        if userask_form.is_valid():
            user_ask = userask_form.save(commit=True)
            return JsonResponse({'status': 'success'})
        else:
            return JsonResponse({'status': 'fail', 'msg': '添加出错'})

class OrgHomeView(View):

    def get(self, request, org_id):
        current_page = 'home'
        course_org = CourseOrg.objects.get(id=int(org_id))
        has_fav = False
        if request.user.is_authenticated():
            if UserFavorite.objects.filter(user=request.user, fav_id=int(course_org.id), fav_type=2):
                has_fav = True
        all_courses = course_org.course_set.all()[:3]
        all_teacher = course_org.teacher_set.all()[:1]
        return render(request, 'org-detail-homepage.html', locals())

class OrgCourseView(View):
    def get(self, request, org_id):
        current_page = 'course'
        course_org = CourseOrg.objects.get(id=int(org_id))
        has_fav = False
        if request.user.is_authenticated():
            if UserFavorite.objects.filter(user=request.user, fav_id=int(course_org.id), fav_type=2):
                has_fav = True
        all_courses = course_org.course_set.all()
        return render(request, 'org-detail-course.html', locals())

class OrgDescView(View):
    def get(self, request, org_id):
        current_page = 'desc'
        course_org = CourseOrg.objects.get(id=int(org_id))
        has_fav = False
        if request.user.is_authenticated():
            if UserFavorite.objects.filter(user=request.user, fav_id=int(course_org.id), fav_type=2):
                has_fav = True
        return render(request, 'org-detail-desc.html', locals())

class OrgTeacherView(View):
    def get(self, request, org_id):
        current_page = 'teacher'
        course_org = CourseOrg.objects.get(id=int(org_id))
        all_teacher = course_org.teacher_set.all()
        has_fav = False
        if request.user.is_authenticated():
            if UserFavorite.objects.filter(user=request.user, fav_id=int(course_org.id), fav_type=2):
                has_fav = True
        return render(request, 'org-detail-teachers.html', locals())

class AddFavView(View):
    '''
    用户收藏
    '''

    def post(self, request):
        fav_id = request.POST.get('fav_id', 0)
        fav_type = request.POST.get('fav_type', 0)

        if not request.user.is_authenticated():
            return JsonResponse({'status': 'fail', 'msg': '用户未登录'})

        exist_records = UserFavorite.objects.filter(user=request.user, fav_id=int(fav_id), fav_type=int(fav_type))
        if exist_records:
            exist_records.delete()
            if int(fav_type) == 1:
                course = Course.objects.get(id=int(fav_id))
                course.fav_nums -= 1
                if course.fav_nums < 0:
                    course.fav_nums = 0
                course.save()
            elif int(fav_type) == 2:
                course_org = CourseOrg.objects.get(id=int(fav_id))
                course_org.fav_nums -= 1
                if course_org.fav_nums < 0:
                    course_org.fav_nums = 0
                course_org.save()
            elif int(fav_type) == 3:
                teacher = Teacher.objects.get(id=int(fav_id))
                teacher.fav_nums -= 1
                if teacher.fav_nums < 0:
                    teacher.fav_nums = 0
                teacher.save()

            return JsonResponse({'status': 'success', 'msg': '收藏'})
        else:
            user_fav = UserFavorite()
            if int(fav_id) > 0 and int(fav_type) > 0:
                user_fav.user = request.user
                user_fav.fav_id = int(fav_id)
                user_fav.fav_type = int(fav_type)
                user_fav.save()
                if int(fav_type) == 1:
                    course = Course.objects.get(id=int(fav_id))
                    course.fav_nums += 1
                    course.save()
                elif int(fav_type) == 2:
                    course_org = CourseOrg.objects.get(id=int(fav_id))
                    course_org.fav_nums += 1
                    course_org.save()
                elif int(fav_type) == 3:
                    teacher = Teacher.objects.get(id=int(fav_id))
                    teacher.fav_nums += 1
                    teacher.save()

                return JsonResponse({'status': 'success', 'msg': '已收藏'})
            return JsonResponse({'status': 'fail', 'msg': '收藏出错'})

class TeacherListView(View):
    def get(self, request):
        all_teacher = Teacher.objects.all()
        # 搜索
        search_keywords = request.GET.get('keywords', '')
        if search_keywords:
            all_teacher = all_teacher.filter(
                Q(name__icontains=search_keywords) | Q(work_company__icontains=search_keywords) | Q(
                    work_position__icontains=search_keywords))

        sort = request.GET.get('sort', '')
        if sort:
            if sort == 'hot':
                all_teacher = all_teacher.order_by('-click_nums')

        sorted_teachers = Teacher.objects.all().order_by('-click_nums')[:3]

        try:
            page = request.GET.get('page', 1)
        except PageNotAnInteger:
            page = 1
        p = Paginator(all_teacher, 1, request=request)
        teachers = p.page(page)
        return render(request, 'teachers-list.html', locals())

class TeacherDetailView(LoginRequiredMixin,View):
    def get(self, request, teacher_id):
        teacher = Teacher.objects.get(id=int(teacher_id))
        all_courses = Course.objects.filter(teacher=teacher)

        has_teacher_faved = False
        if UserFavorite.objects.filter(user=request.user, fav_type=3, fav_id=teacher.id):
            has_teacher_faved = True

        has_org_faved = False
        if UserFavorite.objects.filter(user=request.user, fav_type=2, fav_id=teacher.org.id):
            has_org_faved = True

        sorted_teachers = Teacher.objects.all().order_by('-click_nums')[:3]
        return render(request, 'teacher-detail.html', locals())


import numpy as np
import csv
import os
import glob
import json
import argparse
import cPickle as pickle

caffe_classify_path = '/home/will/code/caffe/python/classify.py'
image_path = '../bin/jpgs/users/'
web_image_path = '../website/static/user_images/'
npy_path = '../bin/npys/users/'
stats_path = '../bin/stats/'
category_path = '../bin/jpgs/categories/'
classifier_path = '../model_data/'

def keywithmaxval(d):
    '''
    input: dictionary
    return: key with the maximum value

    Remove item if key == 'other'.
    '''

    r = dict(d)
    if 'other' in r:
        del r['other']
    v = list(r.values())
    k = list(r.keys())

    return k[v.index(max(v))]

def add_user(username):
    '''
    input: string
    return: None

    Check to see if there is any information already available for the
    user.  If not, scrapes the users images, then sends the images through
    the caffe network and save the resulting set of .npy arrays.
    '''

    # create list of stored user photos
    user_folders = os.listdir(image_path)

    # create list of stored user .npys
    user_npys = os.listdir(npy_path)

    # If we don't have photos, get 'em
    if username not in user_folders:
        os.system(' '.join(['python ../model_code/ScrapeUser.py', username]))
        os.system(' '.join(['cp -r', image_path + username, web_image_path + username]))

    # if we don't have .npys, make 'em
    if username + '.npy' not in user_npys:
        os.system(' '.join(['python ',
                            caffe_classify_path,
                            '--gpu',
                            image_path + username,
                            npy_path + username]))

def unpickle_model(label):
    # unpickle model parts
    with open(classifier_path + label + '_svd.pkl') as fl:
        mod_svd = pickle.load(fl)

    with open(classifier_path + label + '_logit.pkl') as fl:
        mod_logit = pickle.load(fl)

    with open(classifier_path + label + '_labels.csv', 'r') as fl:
        csv_reader = csv.reader(fl)
        categories = list(csv_reader)[0]

    return mod_svd, mod_logit, categories

def run_classifier(label, X):
    '''
    input: string, npy
    return: dict

    Takes a classifier name and an array of image features.
    Returns a dict with the predictions for each image based
    on the specified classifier.
    '''

    # unpickle model parts
    mod_svd, mod_logit, categories = unpickle_model(label)

    # reduce features with SVD
    X_s = mod_svd.transform(X)

    # make predictions with logit
    predictions = mod_logit.predict(X_s)

    # format predictions
    confidence = mod_logit.predict_proba(X_s).max(axis=1)
    preds_c = [categories[int(pred)] for pred in predictions]
    category_counts = dict([(cat, preds_c.count(cat)) for cat in categories])
    top_count = keywithmaxval(category_counts)

    # record predictions in a dict
    label_stats = {}
    label_stats[label + '_predictions'] = preds_c
    label_stats[label + '_confidence'] = confidence.tolist()
    label_stats[label + '_category_counts'] = category_counts
    label_stats[label + '_top_count'] = top_count

    return label_stats

def save_user_stats(username, user_dict, all_users):
    '''
    input: string, dict, dict
    return: None

    Combines a new users stats with the dict of all users stats,
    and saves the result as a json.
    '''
    if os.path.exists(stats_path + 'user_stats.json'):
        with open(stats_path + 'user_stats.json', 'r') as fr:
            old_user_dict = json.load(fr)

        with open(stats_path + 'user_stats.json', 'wb') as fw:
            old_user_dict.update(user_dict)
            json.dump(old_user_dict, fw)

    else:
        with open(stats_path + 'user_stats.json', 'w') as fw:
            json.dump(user_dict, fw)

    # re-pickle updated user data
    all_users.add(username)
    with open(stats_path + 'all_users.pkl', 'w') as fw:
        pickle.dump(all_users, fw)

def predict_user(username):
    '''
    input: username
    return: None

    Takes a username, makes predictions of image categories, and saves the
    predictions to a dictionary with other users.
    '''

    # Scrape user images
    add_user(username)

    # Open the stored user data
    if os.path.exists(stats_path + 'all_users.pkl'):
        with open(stats_path + 'all_users.pkl') as fl:
            all_users = pickle.load(fl)
    else:
        all_users = set()

    # exit if we already have this users predictions
    if username in all_users:
        return

    # load .npy outputs from caffe
    X = np.load(npy_path + username + '.npy')

    # make a list of classifiers to run
    label_folders = os.listdir(category_path)

    # save list of images in the order they are classified
    image_lst = sorted([im_f for im_f in glob.glob(''.join([image_path, username, '/*.jpg']))])

    # run each of the classifiers we have trained
    user_stats = {}
    user_stats['image_lst'] = image_lst

    for label in label_folders:
        user_stats.update(run_classifier(label, X))

    # record stats as a dict with username as key
    user_dict = {}
    user_dict[username] = user_stats

    # Add the new user to the dict of all saved predictions
    save_user_stats(username, user_dict, all_users)

if __name__ == '__main__':
    """
    Usage: python PredictUser.py <username>
    """
    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument('username')

    predict_user(parser.parse_args().username)


# -*- coding: utf-8 -*-
# __author__ = 'sanger'

from __future__ import division
import os

import math

from biocluster.config import Config
from bson.objectid import ObjectId
import types
import json
import numpy as np
import re
from types import StringTypes
import gridfs
from collections import OrderedDict, defaultdict
import pandas as pd
from biocluster.file import getsize, exists
from biocluster.file import download
import shutil
from biocluster.api.file.lib.transfer import MultiFileTransfer
import sys

project_type = 'lnc_rna'
db = Config().get_mongo_client(mtype=project_type)[Config().get_mongo_dbname(project_type)]

# ---------------------基因集及相关gdq----------------------------------
def export_geneset_exp_matrix(data, option_name, dir_path, bind_obj=None):
    """ gdq 2019 04 修改
    该函数根据提供的geneset_id和exp_id和group_dict提取表达信息。
    当geneset_id为‘all’时，则导出所有的exp_id对应的详情表。
    当geneset_id为‘refall’时，则导出所有的exp_id对应的详情表时还需限制'is_new'字段为False(false)。
    该函数还顺便根据group_dict信息生成分组信息文件group_info.txt。
    该函数仅返回表达矩阵文件的路径信息
    """
    exp_id, geneset_id, geneset_type = data.split(";")
    group_dict = bind_obj.sheet.option('group_dict')
    group_dict = json.loads(group_dict, object_pairs_hook=OrderedDict)
    samples = list()
    for each in group_dict:
        samples += group_dict[each]
    # export group info
    with open(dir_path + '/group_info.txt', 'w') as f:
        f.write('#sample\tgroup\n')
        group_id = bind_obj.sheet.option('group_id').lower()
        for g in group_dict:
            for s in group_dict[g]:
                if group_id == "all":
                    g = s
                f.write('{}\t{}\n'.format(s, g))
    # decide output columns
    target_cols = OrderedDict(seq_id=1, _id=0)
    for each in samples:
        target_cols[each] = 1
    # get geneset
    seq_marker = {}
    geneset = set()
    if "all" not in geneset_id.lower():
        conn = db['sg_geneset_detail']
        for gs_id, gs_type in zip(re.split('\s*,\s*', geneset_id), re.split('\s*,\s*', geneset_type)):
            geneset_record = conn.find({"geneset_id": ObjectId(gs_id)})
            if not geneset_record:
                bind_obj.set_error('geneset not found by query: {}'.format(geneset_id))
            sub_geneset = geneset_record[0]['seq_list']
            geneset.update(sub_geneset)
            for seq_id in sub_geneset:
                seq_marker[seq_id] = gs_type

    # get all exp matrix
    conn = db['sg_exp_detail']
    if "refall" in geneset_id.lower():
        exp_records = conn.find({"exp_id": ObjectId(exp_id), 'is_new': False}, target_cols)
    else:
        exp_records = conn.find({"exp_id": ObjectId(exp_id)}, target_cols)
    exp_matrix = pd.DataFrame(list(exp_records))
    if exp_matrix.shape[0] == 0:
        bind_obj.set_error('No expression data find by query: {}'.format(exp_id))
    exp_matrix = exp_matrix.set_index('seq_id')
    if geneset:
        needed_index = exp_matrix.index & geneset
        exp_matrix = exp_matrix.loc[needed_index, :]
    output = os.path.join(dir_path, option_name)
    exp_matrix.to_csv(output, sep='\t', header=True, index=True)

    marker = os.path.join(dir_path, 'seq_id_marker.json')
    with open(marker, 'w') as out_handler:
        json.dump(seq_marker, out_handler, indent=4)

    print('success to export expression matrix')
    return output

def export_geneset_from_query(data, option_name, dir_path, bind_obj=None):
    collection = db['sg_annotation_query_detail']
    main_collection = db['sg_annotation_query']
    my_result = main_collection.find_one({'task_id': data})
    if not my_result:
        bind_obj.set_error("意外错误，task_id:{}在sg_annotation_query中未找到！".format(data))
    if 'main_id' in my_result:
        query_id = my_result["main_id"]
    else:
        query_id = my_result['_id']
    results = collection.find({"query_id": ObjectId(query_id)})
    output = os.path.join(dir_path, "all_genes_transcripts.txt")
    with open(output, "w") as f:
        f.write('transcript_id\tgene_id\n')
        for result in results:
            transcript_id = ''
            if u'transcript_id' in result:
                transcript_id = result['transcript_id']
            gene_id = ''
            if u'gene_id' in result:
                gene_id = result['gene_id']
            f.write('{}\t{}\n'.format(transcript_id, gene_id))
    return output

def export_wgcna_exp_matrix(data, option_name, dir_path, bind_obj=None):
    """ gdq
    该函数仅为wgcna module分析服务。
    该函数会返回wgcna_prepare得到的exp_matrix路径，
    同时还会通过查询sg_annotation_query表导出geneid和genename的关系文件:seq_id2gene_name.txt
    """
    wgcna_prepare = db['sg_wgcna_prepare']
    prepare_id = data.strip()
    prepare_main = wgcna_prepare.find_one({"_id": ObjectId(prepare_id)})
    exp_matrix = prepare_main['exp_matrix']
    if re.match(r'^\w+://\S+/.+$', exp_matrix):
        download_s3_file(exp_matrix, os.path.join(bind_obj.work_dir, "exp_matrix.txt"))
        exp_matrix = os.path.join(bind_obj.work_dir, "exp_matrix.txt")
        if not os.path.exists(os.path.join(dir_path, "exp_matrix.txt")):
            os.link(exp_matrix, os.path.join(dir_path, "exp_matrix.txt"))
    else:
        if os.path.exists(os.path.join(dir_path, "exp_matrix.txt")):
            os.remove(os.path.join(dir_path, "exp_matrix.txt"))
        os.link(exp_matrix, os.path.join(dir_path, "exp_matrix.txt"))
    target_seqs = pd.read_table(exp_matrix, header=0, index_col=0).index
    task_id = prepare_main['task_id']
    annot_table = db['sg_annotation_query']
    try:
        annot_main = annot_table.find_one({"task_id": task_id, "type": "latest", "status": "end"})
    except:
        bind_obj.set_error("cannot find sg_annotation_query main table")
    else:
        if annot_main is None:
            annot_main = annot_table.find_one({"task_id": task_id, "type": "origin", "status": "end"})
    if "main_id" not in annot_main:
        annot_main_id = annot_main['_id']
    else:
        annot_main_id = annot_main['main_id']
    annot_detail = db['sg_annotation_query_detail']
    exp_level = bind_obj.sheet.option('exp_level')
    if exp_level[0].upper() == 'T':
        query_dict = dict(query_id=annot_main_id, )
        result_dict = dict(_id=0, gene_name=1, gene_id=1, transcript_id=1)
        result = annot_detail.find(query_dict, result_dict)
        gene2name = pd.DataFrame(list(result))
        gene2name.set_index('transcript_id', inplace=True)
    else:
        query_dict = dict(query_id=annot_main_id, is_gene=True)
        result_dict = dict(_id=0, gene_name=1, gene_id=1)
        result = annot_detail.find(query_dict, result_dict)
        gene2name = pd.DataFrame(list(result))
        gene2name.set_index('gene_id', inplace=True)
    gene2name = gene2name.loc[list(target_seqs), :]
    gene2name.reset_index(inplace=True)
    output = os.path.join(dir_path, "seq_id2gene_name.txt")
    gene2name.to_csv(output, sep='\t', header=True, index=False)
    gene2name = pd.read_table(output, header=0)
    gene2name.fillna(method="pad", axis=1, inplace=True)
    gene2name.to_csv(output, sep='\t', header=True, index=False)
    return exp_matrix + ';' + output

# added  by gdq for wgcna
def export_wgcna_relate_input(data, option_name, dir_path, bind_obj=None):
    module_id = data.strip()
    # export eigengenes
    eigengenes = db['sg_wgcna_module_eigengenes_detail']
    eigengenes_found = eigengenes.find({"module_id": ObjectId(module_id)}, {"_id": 0, "module_id": 0})
    eigengenes_pd = pd.DataFrame(list(eigengenes_found))
    eigengenes_pd.set_index("module", inplace=True)
    eigengenes_path = os.path.join(dir_path, "module_eigengenes.xls")
    eigengenes_pd.to_csv(eigengenes_path, sep='\t', header=True, index=True)
    # export exp matrix
    prepare_id = db['sg_wgcna_module'].find_one({"main_id": ObjectId(module_id)})['wgcna_prepare_id']
    exp_matrix = db['sg_wgcna_prepare'].find_one({"main_id": ObjectId(prepare_id)})['exp_matrix']
    # export each gene's module and gene_id/gene_name info
    exp_level = bind_obj.sheet.option('exp_level')
    membership = db['sg_wgcna_module_membership_detail']
    query_dict = dict(module_id=ObjectId(module_id))
    return_dict = dict(_id=0, seq_id=1, gene_name=1, module=1, kme=1, block_id=1)
    if exp_level == 'transcript':
        return_dict.update({"gene_id": 1})
    result = membership.find(query_dict, return_dict)
    result_pd = pd.DataFrame(list(result))
    result_pd.set_index("seq_id", inplace=True)
    gene_annot = os.path.join(dir_path, "seq_annot.xls")
    result_pd.to_csv(gene_annot, sep='\t', header=True, index=True)
    return exp_matrix + ';' + eigengenes_path + ";" + gene_annot

def get_after_qc_bam_path(data, option_name, dir_path, bind_obj=None):
    db = Config().get_mongo_client(mtype="lnc_rna")[Config().get_mongo_dbname("lnc_rna")]
    conn = db['sg_specimen']
    result = conn.find({"task_id": data, "about_qc": "after"})
    sorted_result = sorted(result, key=lambda k: k['_id'])
    output = os.path.join(dir_path, option_name)
    with open(output, 'w') as f:
        for i in sorted_result:
            f.write(i['bam_path'] + "\n")
    return output

# ---------------------从蛋白复制过来的------------------------

def export_compare_exp_fc(data, option_name, dir_path, bind_obj=None):
    '''
    导出差异分析accession_id 和 fc
    '''

    diff_id = bind_obj.sheet.option('diff_id')
    compare_group = bind_obj.sheet.option('compare_group')
    target_cols = OrderedDict(seq_id=1, log2fc=1, _id=0)

    bind_obj.logger.debug("导出表达参数 {}".format(target_cols))
    conn = db['sg_diff_detail']
    diff_exp_records = conn.find({"diff_id": ObjectId(diff_id), "compare": compare_group}, target_cols)
    diff_exp_matrix = pd.DataFrame(list(diff_exp_records))
    output = os.path.join(dir_path, option_name)
    diff_exp_matrix.to_csv(output, sep='\t', header=True, index=False)
    print('success to export expression matrix')
    return output

def export_go_enrich_matrix(data, option_name, dir_path, bind_obj=None):
    go_enrich_id = bind_obj.sheet.option('enrich_id')
    go_type = bind_obj.sheet.option('go_type')

    target_cols = OrderedDict(go_id=1, go_type=1, discription=1, p_corrected=1, p_uncorrected=1, seq_list=1, depth=1,
                              _id=0)
    bind_obj.logger.debug("导出GO富集表")
    # get geneset
    conn = db['sg_geneset_go_enrich_detail']
    if go_type == 'ALL' or go_type == 'All':
        go_enrich_records = conn.find({"go_enrich_id": ObjectId(go_enrich_id)}, target_cols)
    else:
        go_enrich_records = conn.find({"go_enrich_id": ObjectId(go_enrich_id), "go_type": go_type}, target_cols)
    go_enrich_matrix = pd.DataFrame(list(go_enrich_records))
    seq_list2 = [i.replace(';', '|') for i in go_enrich_matrix['seq_list']]
    go_enrich_matrix['seq_list'] = seq_list2
    # exp_matrix = exp_matrix.loc[geneset, :]
    output = os.path.join(dir_path, option_name)
    go_enrich_matrix.to_csv(output, sep='\t', header=True, index=False)
    print('success to export go_enrich matrix')
    return output

def export_kegg_enrich_matrix(data, option_name, dir_path, bind_obj=None):
    '''
    导出kegg富集表格
    '''
    kegg_enrich_id = bind_obj.sheet.option('enrich_id')
    target_cols = OrderedDict(_id=0, id=1, term=1, pvalue=1, corrected_pvalue=1, seq_list=1, kegg_type=1)
    bind_obj.logger.debug("导出KEGG富集表")
    # get geneset
    conn = db['sg_geneset_kegg_enrich_detail']
    bind_obj.logger.debug("导出以下列{}".format(target_cols))
    bind_obj.logger.debug("导出以下列{}".format(kegg_enrich_id))
    kegg_enrich_records = conn.find({"kegg_enrich_id": ObjectId(kegg_enrich_id)}, target_cols)
    kegg_enrich_matrix = pd.DataFrame(list(kegg_enrich_records))
    seq_list2 = [i.replace(';', '|') for i in kegg_enrich_matrix['seq_list']]
    kegg_enrich_matrix['seq_list'] = seq_list2
    # exp_matrix = exp_matrix.loc[geneset, :]
    output = os.path.join(dir_path, option_name)
    kegg_enrich_matrix.to_csv(output, sep='\t', header=True, index=False)
    print('success to export kegg_enrich matrix')
    return output

def export_gene_list_ppi(data, option_name, dir_path, bind_obj=None):
    gene_list_path = os.path.join(dir_path, "%s.txt" % option_name)
    bind_obj.logger.debug("正在导出蛋白集")
    collection = db['sg_geneset_detail']
    main_collection = db['sg_geneset']
    my_result = main_collection.find_one({'_id': ObjectId(data)})
    if not my_result:
        bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(data)))
    results = collection.find_one({"geneset_id": ObjectId(data)})["seq_list"]
    with open(gene_list_path, "wb") as f:
        f.write("accession_id" + "\n")
        for result in results:
            f.write(result + "\n")
    bind_obj.logger.debug("蛋白集导出成功！")
    return gene_list_path

def download_s3_file(path, to_path):
    """
    判断文件是否在对象存储上
    """
    if not to_path.startswith("/"):
        to_path = os.path.join(self.work_dir, to_path)
    if os.path.exists(to_path):
        os.remove(to_path)
    elif os.path.exists(path):
        to_path = path
    elif exists(path):
        download(path, to_path)
    else:
        print
        'file can not find'
    return to_path

# added for transcription factor analysis  --------------------------------------------
def get_all_pep_seq(data, option_name, dir_path, bind_obj=None):
    pep_db_path = data.strip()
    result_path = os.path.join(dir_path, "all_pep.fa")
    if re.match(r'^\w+://\S+/.+$', pep_db_path) or re.match(r'/mnt/ilustre', pep_db_path):
        transfer = MultiFileTransfer()
        transfer.add_download(pep_db_path, bind_obj.work_dir + "/")
        transfer.perform()
        # download_s3_file(pep_db_path, os.path.join(bind_obj.work_dir, "pep.db"))
        pep_db_path = os.path.join(bind_obj.work_dir, "refrna_seqs.db")
    bind_obj.logger.debug("文件为 {}".format(pep_db_path))
    import sqlite3
    conn = sqlite3.connect(pep_db_path)
    cursor = conn.cursor()
    cursor.execute("SELECT transcript_id,pep_seq FROM trans_annot")
    with open(result_path, 'w') as fw:
        # pep_id 就是转录本的id
        for pep_id, pep_seq in cursor.fetchall():
            fw.write('>{}\n{}\n'.format(pep_id, pep_seq))
    return result_path

def get_gene_detail(data, option_name, dir_path, bind_obj=None):
    task_id = data.strip()
    annot_table = db['sg_annotation_query']
    annot_main = annot_table.find_one({"task_id": task_id, "type": "latest", 'status': 'end'})
    if not annot_main:
        annot_main = annot_table.find_one({"task_id": task_id, "type": "origin"})
    if not annot_main:
        task_table = db['sg_task']
        task_main = task_table.find_one({"task_id": task_id})
        if os.path.exists(task_main['annot']):
            gene_annot = os.path.join(dir_path, "seq_annot.xls")
            os.system("cut -f 1-4 " + task_main['annot'] + " |awk '{print $2\"\\t\"$1\"\\t\"$4}' > " + gene_annot)
            return gene_annot
            # os.system("cut -f 1-4 {} |awk '{print $2\"\t\"$1\"\t\"$4}' > {} ".format(task_main['annot'], gene_annot))
        else:
            bind_obj.set_error("Not Found in sg_annotation_query by query {}".format(task_id))
    if "main_id" not in annot_main:
        annot_main_id = annot_main['_id']
    else:
        annot_main_id = annot_main['main_id']
    annot_detail = db['sg_annotation_query_detail']
    query_dict = dict(query_id=annot_main_id, )
    result_dict = dict(_id=0, gene_name=1, gene_id=1, description=1, transcript_id=1)
    result = annot_detail.find(query_dict, result_dict)
    result_pd = pd.DataFrame(list(result))
    result_pd.set_index("transcript_id", inplace=True)
    result_pd = result_pd.loc[:, ["gene_id", "gene_name", "description"]]
    result_pd.columns = ["gene_id", "gene_name", "gene_desc"]
    # result_pd = result_pd.loc[:, ["gene_id", "gene_name"]]
    # result_pd.columns = ["gene_id", "gene_name"]
    result_pd["gene_name"][result_pd['gene_name'] == "-"] = None
    result_pd = result_pd.fillna(method="pad", axis=1)
    gene_annot = os.path.join(dir_path, "seq_annot.xls")
    result_pd.to_csv(gene_annot, sep='\t', header=True, index=True)
    os.system(
        r"sed -i 's/%2B/+/g;s/%2F/\//g;s/%2C/,/g;s/%3A/:/g;s/%3B/;/g;s/%3D/=/g;s/%3F/?/g;s/%20/ /g;s/%25/%/g;s/%3C/</g;s/%3E/>/g;s/%5B/[/g;s/%5D/]/g;s/%7B/{/g;s/%7D/}/g' " + gene_annot)
    return gene_annot

# ---------------------基因集及相关zhaozhipeng----------------------------------
def export_genesets_exp_matrix(data, option_name, dir_path, bind_obj=None):
    """
    该函数根据提供的geneset_id和exp_id和group_dict提取表达信息。
    当geneset_id为‘all’时，则导出所有的exp_id对应的详情表。
    当geneset_id为‘refall’时，则导出所有的exp_id对应的详情表时还需限制'is_new'字段为False(false)。
    该函数还顺便根据group_dict信息生成分组信息文件group_info.txt。
    该函数仅返回表达矩阵文件的路径信息
    """
    s_list = data.split(";")
    exp_id = s_list[0]
    geneset_ids = s_list[1:]
    group_dict = bind_obj.sheet.option('group_dict')
    group_dict = json.loads(group_dict, object_pairs_hook=OrderedDict)
    samples = list()
    for each in group_dict:
        samples += group_dict[each]
    # export group info
    with open(dir_path + '/group_info.txt', 'w') as f:
        f.write('#sample\tgroup\n')
        group_id = bind_obj.sheet.option('group_id').lower()
        for g in group_dict:
            for s in group_dict[g]:
                if group_id == "all":
                    g = s
                f.write('{}\t{}\n'.format(s, g))
    # decide output columns
    target_cols = OrderedDict(seq_id=1, _id=0)
    for each in samples:
        target_cols[each] = 1
    # get geneset
    geneset = set()
    bind_obj.logger.debug('=========== %s ++++++++++++++' % ','.join(geneset_ids))
    for geneset_id in geneset_ids:
        if "all" not in geneset_id.lower():
            conn = db['sg_geneset_detail']
            geneset_records = conn.find_one({"geneset_id": ObjectId(geneset_id)})
            if not geneset_records:
                bind_obj.set_error('geneset not found by query: {}'.format(geneset_id))
            geneset.update(geneset_records['seq_list'])

    # get all exp matrix
    conn = db['sg_exp_detail']
    exp_records = conn.find({"exp_id": ObjectId(exp_id)}, target_cols)
    exp_matrix = pd.DataFrame(list(exp_records))
    if exp_matrix.shape[0] == 0:
        bind_obj.set_error('No expression data find by query: {}'.format(exp_id))

    exp_matrix = exp_matrix.set_index('seq_id')
    if geneset:
        indexs = exp_matrix.index & {i for i in geneset}
        exp_matrix = exp_matrix.loc[indexs, :]
    exp_matrix.dropna(how='all')
    output = os.path.join(dir_path, option_name)
    exp_matrix.to_csv(output, sep='\t', header=True, index=True)
    print('success to export expression matrix')
    return output

def export_genes_list(data, option_name, dir_path, bind_obj=None):
    """

    :param data:
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug('=========== %s ++++++++++++++' % data)
    geneset_ids = [item.split(',') for item in data.split(";")]
    geneset = {}
    for gene_type, geneset_id in geneset_ids:
        if "all" not in geneset_id.lower():
            conn = db['sg_geneset_detail']
            geneset_records = conn.find_one({"geneset_id": ObjectId(geneset_id)})
            if not geneset_records:
                bind_obj.set_error('geneset not found by query: {}'.format(geneset_id))
            for i in geneset_records['seq_list']:
                geneset[i] = 'lncRNA' if gene_type == 'l_geneset_id' else 'mRNA'

    output = os.path.join(dir_path, 'genes_info.json')
    with open(output, 'w') as in_handler:
        json.dump(geneset, in_handler, indent=4)

    return output

def export_diff_exp_matrix(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'diff_group': 'xxx|yyy', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export diff_exp.txt file')
    compare = data['diff_group']
    sg_diff_id = data['sg_diff_id']

    conn = db['sg_diff_detail']
    recoreds = conn.find({'diff_id': ObjectId(sg_diff_id), 'compare': compare})
    res_data = []
    for dic in recoreds:
        tmp_dic = {'seq_id': dic['seq_id'], 'log2fc': dic['log2fc'], 'regulate': dic['regulate'],
                   'avg_exp': (dic['group1'] + dic['group2']) / 2, 'significant': dic['significant']}
        res_data.append(tmp_dic)
    result_df = pd.DataFrame(res_data)
    output = os.path.join(dir_path, 'diff_exp.txt')
    result_df.to_csv(output, sep='\t', index=False)
    bind_obj.logger.debug(' :: export diff_exp.txt file :: completed')

    return output

def export_rna_type_matrix(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'task_id': 'lnc_rna', 'exp_level': 'T', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export seq_id_type.txt file')
    task_id = data['task_id']
    sg_diff_id = data['diff_exp']
    exp_level = data['exp_level']
    # {'task_id': data.task_id, 'exp_level': 'T', 'diff_exp': data.diff_exp}
    conn = db['sg_diff']
    bind_obj.logger.debug(str({'main_id': ObjectId(sg_diff_id), 'exp_level': exp_level, 'task_id': task_id}))
    # recored = conn.find_one({'main_id': ObjectId(sg_diff_id), 'exp_level': exp_level, 'task_id': task_id})
    # 因为main_id本身唯一,所以删除多余查找字段
    recored = conn.find_one({'main_id': ObjectId(sg_diff_id)})
    exp_id = ObjectId(recored['exp_id'])

    conn = db['sg_exp_detail']
    recoreds = conn.find({'exp_id': exp_id}, {'seq_id': 1, 'rna_type': 1, '_id': 0})
    result_df = pd.DataFrame([i for i in recoreds])
    output = os.path.join(dir_path, 'seq_id_type.txt')
    result_df.to_csv(output, sep='\t', index=False)
    bind_obj.logger.debug(' :: export seq_id_type.txt file :: completed')

    return output

def export_cis_matrix2(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'task_id': 'lnc_rna', 'exp_level': 'T', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export cis_targets.txt file')

    # 获取基因转录本对应关系， 只要一组

    task_id = bind_obj.sheet.option("task_id")
    annot_table = db['sg_annotation_query']
    annot_main = annot_table.find_one({"task_id": task_id, "type": "origin", "status": "end"})
    annot_detail = db['sg_annotation_query_detail']
    result = annot_detail.find({"query_id": annot_main["main_id"], "is_gene": True},
                               {"_id": 0, "transcript_id": 1, "gene_id": 1})
    g2t_df = pd.DataFrame(list(result))

    conn = db['sg_target_cistrans_detail']
    recoreds = conn.find({'target_cistrans_id': ObjectId(data), "target_type": "cis"},
                         {'lncrna_id': 1, 'gene_id': 1, 'mgene_id': 1, '_id': 0})
    data = [i for i in recoreds]
    output = os.path.join(dir_path, 'cis_targets.txt')
    if data:
        result_df = pd.DataFrame(data)
        result_df = result_df.merge(g2t_df, left_on = 'mgene_id', right_on = 'gene_id', suffixes=("", "_ri"))
        result_df.drop(columns=["gene_id_ri"],  inplace=True)
        result_df.rename(columns={'transcript_id': "mrna_id"}, inplace=True)
        result_df.to_csv(output, sep='\t', index=False)
    else:
        bind_obj.logger.debug('cis_targets.txt 文件为空, %s' % len(data))
        with open(output, 'w') as out_handler:
            out_handler.write('lncrna_id\tgene_id\tmrna_id\tmgene_id\n')
    bind_obj.logger.debug(' :: export cis_targets.txt file :: completed')

    return output

def export_cis_matrix(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'task_id': 'lnc_rna', 'exp_level': 'T', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export cis_targets.txt file')

    conn = db['sg_target_cis_detail']
    recoreds = conn.find({'target_cis_id': ObjectId(data)},
                         {'lncrna_id': 1, 'gene_id': 1, 'mrna_id': 1, 'mgene_id': 1, '_id': 0})
    data = [i for i in recoreds]
    output = os.path.join(dir_path, 'cis_targets.txt')
    if data:
        result_df = pd.DataFrame(data)

        result_df.to_csv(output, sep='\t', index=False)
    else:
        bind_obj.logger.debug('cis_targets.txt 文件为空, %s' % len(data))
        with open(output, 'w') as out_handler:
            out_handler.write('lncrna_id\tgene_id\tmrna_id\tmgene_id\n')
    bind_obj.logger.debug(' :: export cis_targets.txt file :: completed')

    return output

def export_trans_matrix2(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'task_id': 'lnc_rna', 'exp_level': 'T', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export trans_targets.txt file')

    # 获取基因转录本对应关系， 只要一组

    task_id = bind_obj.sheet.option("task_id")
    annot_table = db['sg_annotation_query']
    annot_main = annot_table.find_one({"task_id": task_id, "type": "origin", "status": "end"})
    annot_detail = db['sg_annotation_query_detail']
    result = annot_detail.find({"query_id": annot_main["main_id"], "is_gene": True},
                               {"_id": 0, "transcript_id": 1, "gene_id": 1})
    g2t_df = pd.DataFrame(list(result))

    conn = db['sg_target_cistrans_detail']
    recoreds = conn.find({'target_cistrans_id': ObjectId(data), "target_type": "trans"},
                         {'lncrna_id': 1, 'gene_id': 1, 'mgene_id': 1, '_id': 0})

    data = []
    check = defaultdict(int)
    for recored in recoreds:
        lnc_id = recored['lncrna_id']
        num = check[lnc_id]
        if num >= 3:
            continue
        check[lnc_id] += 1
        data.append(recored)

    output = os.path.join(dir_path, 'trans_targets.txt')
    if data:
        result_df = pd.DataFrame(data)
        result_df = result_df.merge(g2t_df, left_on = 'mgene_id', right_on = 'gene_id', suffixes=("", "_ri"))
        result_df.drop(columns=["gene_id_ri"],  inplace=True)
        result_df.rename(columns={'transcript_id': "mrna_id"}, inplace=True)

        result_df.to_csv(output, sep='\t', index=False)
    else:
        bind_obj.logger.debug('trans_targets.txt 文件为空, %s' % len(data))
        with open(output, 'w') as out_handler:
            out_handler.write('lncrna_id\tgene_id\tmrna_id\tmgene_id\n')
    bind_obj.logger.debug(' :: export trans_targets.txt file :: completed')

    return output

def export_trans_matrix(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {'task_id': 'lnc_rna', 'exp_level': 'T', 'sg_diff': '5c9da1ba17b2bf206a55d350'}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    bind_obj.logger.debug(' :: export trans_targets.txt file')

    conn = db['sg_target_trans_detail']
    recoreds = conn.find({'target_trans_id': ObjectId(data)},
                         {'lncrna_id': 1, 'gene_id': 1, 'mrna_id': 1, 'mgene_id': 1, '_id': 0})
    data = []
    check = defaultdict(int)
    for recored in recoreds:
        lnc_id = recored['lncrna_id']
        num = check[lnc_id]
        if num >= 3:
            continue
        check[lnc_id] += 1
        data.append(recored)

    output = os.path.join(dir_path, 'trans_targets.txt')
    if data:
        result_df = pd.DataFrame(data)
        result_df.to_csv(output, sep='\t', index=False)
    else:
        bind_obj.logger.debug('trans_targets.txt 文件为空, %s' % len(data))
        with open(output, 'w') as out_handler:
            out_handler.write('lncrna_id\tgene_id\tmrna_id\tmgene_id\n')
    bind_obj.logger.debug(' :: export trans_targets.txt file :: completed')

    return output

def export_multi_gene_list(data, option_name, dir_path, bind_obj=None):
    data = json.loads(data)
    geneset_id_list = data['geneset_id'].split(",")
    source = data['source']
    multi_geneset_path = dir_path + "/multi_geneset_list"
    main_collection = db['sg_geneset']
    collection = db['sg_geneset_detail']
    with open(multi_geneset_path, "wb") as out_handler:
        if len(geneset_id_list) == 1 and source == 'diff_exp':
            geneset_id = geneset_id_list[0]
            my_result = main_collection.find_one({'main_id': ObjectId(geneset_id)})
            if not my_result:
                bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(geneset_id)))
            geneset_name = my_result["name"]
            results = collection.find_one({"geneset_id": ObjectId(geneset_id)})
            seq_list = results["seq_list"]
            if 'regulate_list' not in results:
                raise bind_obj.set_error(
                    'geneset_detail table must contain "regulate_list" field when provide one geneset id')
            regulate_list = results["regulate_list"]
            up_list, down_list = [], []
            for seq_id, regulate in zip(seq_list, regulate_list):
                if regulate == 'up':
                    up_list.append(seq_id)
                elif regulate == 'down':
                    down_list.append(seq_id)
                else:
                    continue
            out_handler.write(geneset_name + '_up\t' + ','.join(up_list) + '\n')
            out_handler.write(geneset_name + '_down\t' + ','.join(down_list) + '\n')
        else:
            for n, gi in enumerate(geneset_id_list):
                my_result = main_collection.find_one({'main_id': ObjectId(gi)})
                if not my_result:
                    bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(gi)))
                out_handler.write(my_result["name"] + "\t")
                results = collection.find_one({"geneset_id": ObjectId(gi)})
                out_handler.write(",".join(results["seq_list"]) + "\n")

    return multi_geneset_path

def export_go_detail(data, option_name, dir_path, bind_obj=None):
    geneset_id_list = data.split(",")
    multi_geneset_path = dir_path + "/multi_geneset_list"
    main_collection = db['sg_geneset']
    collection = db['sg_geneset_detail']
    with open(multi_geneset_path, "wb") as out_handler:
        if len(geneset_id_list) == 2:
            for n, gi in enumerate(geneset_id_list):
                my_result = main_collection.find_one({'main_id': ObjectId(gi)})
                if not my_result:
                    bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(gi)))
                out_handler.write(my_result["name"] + "\t")
                results = collection.find_one({"geneset_id": ObjectId(gi)})
                out_handler.write(",".join(results["seq_list"]) + "\n")
        else:
            geneset_id = geneset_id_list[0]
            my_result = main_collection.find_one({'main_id': ObjectId(geneset_id)})
            if not my_result:
                bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(geneset_id)))
            geneset_name = my_result["name"]
            results = collection.find_one({"geneset_id": ObjectId(geneset_id)})
            seq_list = results["seq_list"]
            if 'regulate_list' not in results:
                raise bind_obj.set_error(
                    'geneset_detail table must contain "regulate_list" field when provide one geneset id')
            regulate_list = results["regulate_list"]
            up_list, down_list = [], []
            for seq_id, regulate in zip(seq_list, regulate_list):
                if regulate == 'up':
                    up_list.append(seq_id)
                else:
                    down_list.append(seq_id)
            out_handler.write(geneset_name + '_up\t' + ','.join(up_list) + '\n')
            out_handler.write(geneset_name + '_down\t' + ','.join(down_list) + '\n')

    return multi_geneset_path

def export_kegg_detail(data, option_name, dir_path, bind_obj=None):
    """

    :param data: {''}
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    geneset_id_list = data.split(",")
    multi_geneset_path = dir_path + "/multi_geneset_list"
    main_collection = db['sg_geneset']
    collection = db['sg_geneset_detail']
    with open(multi_geneset_path, "wb") as out_handler:
        if len(geneset_id_list) == 2:
            for n, gi in enumerate(geneset_id_list):
                my_result = main_collection.find_one({'main_id': ObjectId(gi)})
                if not my_result:
                    bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(gi)))
                out_handler.write(my_result["name"] + "\t")
                results = collection.find_one({"geneset_id": ObjectId(gi)})
                out_handler.write(",".join(results["seq_list"]) + "\n")
        else:
            geneset_id = geneset_id_list[0]
            my_result = main_collection.find_one({'main_id': ObjectId(geneset_id)})
            if not my_result:
                bind_obj.set_error("意外错误，geneset_id:{}在sg_geneset中未找到！".format(ObjectId(geneset_id)))
            geneset_name = my_result["name"]
            results = collection.find_one({"geneset_id": ObjectId(geneset_id)})
            seq_list = results["seq_list"]
            if 'regulate_list' not in results:
                bind_obj.set_error(
                    'geneset_detail table must contain "regulate_list" field when provide one geneset id')
            regulate_list = results["regulate_list"]
            up_list, down_list = [], []
            for seq_id, regulate in zip(seq_list, regulate_list):
                if regulate == 'up':
                    up_list.append(seq_id)
                else:
                    down_list.append(seq_id)
            out_handler.write(geneset_name + '_up\t' + ','.join(up_list) + '\n')
            out_handler.write(geneset_name + '_down\t' + ','.join(down_list) + '\n')

    return multi_geneset_path

def export_enrich_heatmap(data, option_name, dir_path, bind_obj=None):
    """
        ['task_id', 'submit_location', 'geneset_type', 'enrich_type',
        'geneset_enrich_ids', 'go_level', 'task_type', 'pvalue_type', 'top_num', 'ids_list', 'go_level']

    :param data:
    :param option_name:
    :param dir_path:
    :param bind_obj:
    :return:
    """
    ids_filer_list = None
    if 'ids_list' in data:
        ids_filer_list = {i for i in data['ids_list']}
    query_dict = {}
    filter_dict = dict(_id=0)
    out_file = os.path.join(dir_path, 'heatmap_data.xls')

    if data['enrich_type'].lower() == 'kegg':
        if data['pvalue_type'].lower() == 'pvalue':
            # if ids_filer_list is None:
            #     query_dict['pvalue'] = {'$lte': data['p_value']}
            filter_dict['pvalue'] = 1
            p_field = 'pvalue'
        else:
            # if ids_filer_list is None:
            #     query_dict['corrected_pvalue'] = {'$lte': data['p_value']}
            filter_dict['corrected_pvalue'] = 1
            p_field = 'corrected_pvalue'

        main_collection = db['sg_geneset_kegg_enrich']
        collection = db['sg_geneset_kegg_enrich_detail']
        ids = [ObjectId(i) for i in data['geneset_enrich_ids'].strip().split(',')]
        filter_dict['id'] = 1
        filter_dict['kegg_type'] = 1
        filter_dict['discription'] = 1

        data_list = {}
        detail_info = defaultdict(dict)
        needed_ids = set()
        geneset_names= []
        min_non_zero = 1
        has_zero = False
        for id_obj in ids:
            query_dict['kegg_enrich_id'] = id_obj
            name = main_collection.find_one({'_id': id_obj})['geneset_name']
            geneset_names.append(name)
            sub_data = {}
            data_list[name] = sub_data
            for dic in collection.find(query_dict, filter_dict):
                kegg_id = dic['id']
                sub_dic = detail_info[kegg_id]
                if 'kegg_type' not in sub_dic:
                    sub_dic['kegg_type'] = dic.pop('kegg_type')
                    sub_dic['discription'] = dic.pop('discription')
                p_value = dic[p_field]
                sub_data[kegg_id] = p_value
                if p_value == 0:
                    has_zero = True
                else:
                    min_non_zero = min(p_value, min_non_zero)
            if ids_filer_list is None:
                # 过滤top num 的ids
                top_num_ids = {t for t in
                               sorted(((k, v) for k, v in sub_data.items()), key=lambda items: items[1])[
                               : data['top_num']]}
                # 过滤p_value值
                p_filter_ids = {t[0] for t in top_num_ids if t[1] <= data['p_value']}
                # 取所有ids的并集
                needed_ids.update(p_filter_ids)

        df = pd.DataFrame(data_list)
        ids_filer_list = ids_filer_list or needed_ids
        if len(ids_filer_list) == 0:
            bind_obj.set_error('提供的id集合和基因集没有交集，请重新输入参数')
        if len(geneset_names) != len(set(geneset_names)):
            bind_obj.set_error('提供的id集合名称重复：' + str(geneset_names))

        df = df.loc[df.index & ids_filer_list]
        if has_zero:  # 0 值取最小正实数的十分之一 [比最小正实数小一个数量级]
            df[df == 0] = min_non_zero / 10
        df.fillna(1, inplace=True)
        df = -np.log10(df)
        df[df == 0] = 0
        indexs = df.index
        df['kegg_type'] = [detail_info[i]['kegg_type'] for i in indexs]
        df['discription'] = [detail_info[i]['discription'] for i in indexs]

        df.index.name = 'kegg_id'
        df.to_csv(out_file, sep='\t', index=True, header=True)
    else:
        if data['pvalue_type'].lower() == 'pvalue':
            # if ids_filer_list is None:
            #     query_dict['p_uncorrected'] = {'$lte': data['p_value']}
            filter_dict['p_uncorrected'] = 1
            p_field = 'p_uncorrected'
        else:
            # if ids_filer_list is None:
            #     query_dict['p_corrected'] = {'$lte': data['p_value']}
            filter_dict['p_corrected'] = 1
            p_field = 'p_corrected'

        main_collection = db['sg_geneset_go_enrich']
        collection = db['sg_geneset_go_enrich_detail']
        ids = [ObjectId(i) for i in data['geneset_enrich_ids'].strip().split(',')]
        filter_dict['go_id'] = 1
        filter_dict['go_type'] = 1
        filter_dict['discription'] = 1
        if 'go_level' in data:
            go_level = data['go_level']
            if go_level.lower() != 'all':
                query_dict['go_type'] = go_level.upper()

        data_list = {}
        detail_info = defaultdict(dict)
        needed_ids = set()
        geneset_names= []
        min_non_zero = 1
        has_zero = False
        for id_obj in ids:
            query_dict['go_enrich_id'] = id_obj
            name = main_collection.find_one({'_id': id_obj})['geneset_name']
            geneset_names.append(name)
            sub_data = {}
            data_list[name] = sub_data
            print(query_dict, '========================')
            for dic in collection.find(query_dict, filter_dict):
                go_id = dic['go_id']
                sub_dic = detail_info[go_id]
                if 'go_type' not in sub_dic:
                    sub_dic['go_type'] = dic.pop('go_type')
                    sub_dic['discription'] = dic.pop('discription')
                p_value = dic[p_field]
                sub_data[go_id] = p_value
                if p_value == 0:
                    has_zero = True
                else:
                    min_non_zero = min(p_value, min_non_zero)

            if ids_filer_list is None:
                # 过滤top num 的ids
                top_num_ids = {t for t in
                               sorted(((k, v) for k, v in sub_data.items()), key=lambda items: items[1])[
                               : data['top_num']]}
                # 过滤p_value值
                p_filter_ids = {t[0] for t in top_num_ids if t[1] <= data['p_value']}
                # 取所有ids的并集
                needed_ids.update(p_filter_ids)

        df = pd.DataFrame(data_list)
        ids_filer_list = ids_filer_list or needed_ids
        if len(ids_filer_list) == 0:
            bind_obj.set_error('提供的id集合和基因集没有交集，请重新输入参数')
        df = df.loc[df.index & ids_filer_list]
        if has_zero:  # 0 值取最小正实数的十分之一
            df[df == 0] = min_non_zero / 10
        df.fillna(1, inplace=True)
        df = -np.log10(df)
        df[df == 0] = 0
        indexs = df.index
        df['go_type'] = [detail_info[i]['go_type'] for i in indexs]
        df['discription'] = [detail_info[i]['discription'] for i in indexs]
        df.fillna(1, inplace=True)

        df.index.name = 'go_id'
        df.to_csv(out_file, sep='\t', index=True, header=True)

    return out_file


import sys
sys.stdin = open('input.txt')
from collections import deque

dx = [-1, 0, 1, 0]
dy = [0, 1, 0, -1]

def bfs():
    que = deque()
    que.append((0, 0))
    v[0][0] = 1
    cnt = 0

    while que:
        x, y = que.popleft()
        for d in range(4):
            nx = x + dx[d]
            ny = y + dy[d]
            if 0 <= nx < X and 0 <= ny < Y and not v[nx][ny]:
                if not data[nx][ny]:
                    que.append((nx, ny))
                    v[nx][ny] = 1
                else:
                    data[nx][ny] = 0
                    v[nx][ny] = 1
                    cnt += 1

    return cnt

X, Y = map(int, input().split())
data = [list(map(int, input().split())) for _ in range(X)]

ans = []
time = 0
while True:
    v = [[0]*Y for _ in range(X)]

    cnt = bfs()
    ans.append(cnt)

    if not cnt:
        break

    time += 1

print(time)
print(ans[-2])


import pytest

from mcanitexgen.animation.parser import Duration, ParserError, Time, Timeframe, Weight

class Test_Timeframe_init:
    @pytest.mark.parametrize(
        "start, end, duration, expected_timeframe",
        [
            # Deduce end and duration
            (0, None, None, Timeframe(0, 1, 1)),
            (10, None, None, Timeframe(10, 11, 1)),
            # Deduce duration
            (0, 20, None, Timeframe(0, 20, 20)),
            (11, 22, None, Timeframe(11, 22, 11)),
            # Deduce end
            (0, None, 5, Timeframe(0, 5, 5)),
            (15, None, 5, Timeframe(15, 20, 5)),
            # All set
            (0, 10, 10, Timeframe(0, 10, 10)),
        ],
    )
    def test_args(self, start, end, duration, expected_timeframe):
        assert Timeframe(start, end, duration) == expected_timeframe

    @pytest.mark.parametrize(
        "start, end, duration, match",
        [
            (None, None, None, "Timeframe must have at least one of start, end, duration set"),
            (None, 2, 20, "Timeframes without start can't have end and duration"),
            (0, 5, 20, "Start, end and duration of timeframe don't match: 0, 5, 20"),
        ],
    )
    def test_illegal_args(self, start, end, duration, match):
        with pytest.raises(ParserError, match=match):
            Timeframe(start, end, duration)

class Test_Time_from_args:
    @pytest.mark.parametrize(
        "start, end, duration, weight, expected_time",
        [
            (None, None, None, None, None),
            # Weight
            (None, None, None, 12, Weight(12)),
            # Duration
            (None, None, 10, None, Duration(10)),
            # Timeframe
            (0, None, None, None, Timeframe(0, 1, 1)),
            (1, 20, None, None, Timeframe(1, 20, 19)),
            (1, 20, 19, None, Timeframe(1, 20, 19)),
            (1, None, 19, None, Timeframe(1, 20, 19)),
            (None, 10, None, None, Timeframe(None, 10, None)),
        ],
    )
    def test_args(self, start, end, duration, weight, expected_time):
        assert Time.from_args(start, end, duration, weight) == expected_time

    @pytest.mark.parametrize(
        "start, end, duration, weight, match",
        [
            # Weight
            (None, None, None, 0, "Weight of time must be at least 1"),
            (None, None, 1, 1, "Weighted time can't have start, end or duration"),
            (None, 1, None, 1, "Weighted time can't have start, end or duration"),
            (1, None, None, 1, "Weighted time can't have start, end or duration"),
            # Duration
            (None, None, 0, None, "Duration must be at least 1"),
            (None, None, -10, None, "Duration must be at least 1"),
        ],
    )
    def test_illegal_args(self, start, end, duration, weight, match):
        with pytest.raises(ParserError, match=match):
            Time.from_args(start, end, duration, weight)


class Solution:
    def maxScore(self, nums: List[int]) -> int:
        allVisited = (1 << (len(nums)+1))-1
        cache = {}

        def dp(curr, visited):
            if (curr, visited) in cache:
                return cache[(curr, visited)]
            ans = 0
            for i in range(len(nums)):
                if visited & (1 << i) == 0:
                    for j in range(len(nums)):
                        if visited & (1 << j) == 0 and i != j:
                            newVisited = visited | (1<<i)
                            newVisited = newVisited | (1 <<j)
                            ans = max(ans, curr*(math.gcd(nums[i], nums[j])) + dp(curr+1, newVisited))

            cache[(curr, visited)] = ans
            return ans

        return dp(1, 0)

1799. Maximize Score After N Operations.py


"""ObjectChoice type definition."""

import inspect
from dataclasses import dataclass, field
from functools import cache
from types import ModuleType
from typing import Any

from aiovantage.config_client import models

def get_all_module_classes(module: ModuleType) -> list[type[Any]]:
    """Get all classes from a module."""
    classes = []
    for _, cls in inspect.getmembers(module, inspect.isclass):
        classes.append(cls)

    return classes

@cache
def get_all_object_choices(module: ModuleType) -> list[dict[str, Any]]:
    """Get all object choices from a module."""
    choices = []
    for cls in get_all_module_classes(module):
        # Get the name of the XML element
        meta = getattr(cls, "Meta", None)
        name = getattr(meta, "name", cls.__qualname__)

        # Add the xsdata choice
        choices.append({"name": name, "type": cls})

    return choices

@dataclass
class ObjectChoice:
    """Wildcard type that can be used to represent any object."""

    id: int = field(
        metadata={
            "name": "VID",
            "type": "Attribute",
        }
    )

    choice: object = field(
        metadata={
            "type": "Wildcard",
            "choices": get_all_object_choices(models),
        },
    )


from django.urls import reverse, resolve

from utils.test_bases import ProjectsTestBase
from users import views

class DeleteProjectTest(ProjectsTestBase):
    def test_delete_project_view_function(self):
        view = resolve(reverse('users:delete_project'))

        self.assertIs(view.func, views.delete_project)

    def test_delete_project_succesful(self):
        self.make_project_and_login()
        response = self.base_test_function(
            'users:delete_project', method='post', data={'project_id': 1}
        )
        msg = 'Projeto "Project title" deletado com sucesso!'

        self.assertIn(msg, response.content.decode('utf-8'))

    def test_delete_project_receive_get_method(self):
        self.make_project_and_login()
        response = self.base_test_function('users:delete_project')

        self.assertEqual(response.status_code, 404)

    def test_project_doesnt_exist(self):
        self.register_and_login()
        response = self.base_test_function(
            'users:delete_project', method='post', data={'project_id': 1}
        )

        self.assertEqual(response.status_code, 404)


"""
Given two binary strings, return their sum (also a binary string).

The input strings are both non-empty and contains only characters 1 or 0.

Example 1:

Input: a = "11", b = "1"
Output: "100"
Example 2:

Input: a = "1010", b = "1011"
Output: "10101"
"""

class Solution:
    def addBinary(self, a, b):
        """
        :type a: str
        :type b: str
        :rtype: str
        """
        return bin(int(a,2) + int(b,2))[2:]

    def addBinary2(self,a,b):

        if len(a) < len(b):
            a,b = b,a

        result,carry,power = 0,0,1
        for i in range(max(len(a),len(b))):

            if i >= len(b):
                value = carry + int(a[-(i+1)])
            else:
                value = carry + int(a[-(i+1)]) + int(b[-(i+1)])

            carry,digit = divmod(value,2)

            result = power * digit + result
            power = power * 10

        result =  power * carry + result

        return str(result)

from itertools import zip_longest
class Solution:
    def addBinary(self, a, b):
        """
        :type a: str
        :type b: str
        :rtype: str
        """
        ans,carry = "",0
        for a,b in zip_longest(a[::-1],b[::-1],fillvalue=0):
            carry,digit = divmod(int(a)+int(b)+carry,2)
            ans = str(digit) + ans
        if carry:
            ans = str(carry) + ans
        return ans

class Solution:
    def addBinary(self, a: str, b: str) -> str:

        res = []

        a_bits = list(map(int, a))
        b_bits = list(map(int, b))

        if len(a_bits) < len(b_bits):
            b_bits, a_bits = a_bits, b_bits

        # 使用一个异或门来产生 S，一个与门来产生 C
        def half_adder(a, b):
            return a ^ b, a & b

        # 全加器可以用两个半加器来构造，将输入端A和B连接到一个半加器上，然后将其和输出信号与进位输入信号分别作为第二个半加器的两个输入，并将两个进位输出信号进行逻辑或运算。全加器的关键路径（critical path，即经历最多逻辑门的路径）经过两个异或门，终止于和位s。假定异或门耗费3个延迟来完成，一个全加器的关键路径上施加的延迟等于
        def full_adder(a, b, c_in):
            s1, c = half_adder(a, b)
            s2, c_out = half_adder(s1, c_in)
            return s2, c | c_out

        c_in = 0
        i = 1
        while i <= len(b_bits):
            b, c_in = full_adder(a_bits[-i], b_bits[-i], c_in)
            print(b,c_in)
            i+=1
            res.append(str(b))

        while i <= len(a_bits):
            b, c_in = full_adder(a_bits[-i], 0, c_in)
            i += 1
            res.append(str(b))

        if c_in:
            res.append(str(c_in))

        return "".join(res)[::-1]

if __name__ == '__main__':
    print(Solution().addBinary2(a = "111", b = "1010"))