Validate candle csv files

2023-05-31 10:05:45 +09:00
parent b03553a19d
commit ee76299ac9
7 changed files with 210 additions and 56 deletions
--- a/KissMe/Sources/Domestic/DomesticStock.swift
+++ b/KissMe/Sources/Domestic/DomesticStock.swift
@@ -9,6 +9,11 @@ import Foundation


 public struct Domestic {
+    public typealias Candle = MinutePriceResult.OutputPrice
+}
+
+
+extension Domestic {
    
    /// 국내주식주문 - 주식주문(현금)
    ///
--- a/KissMe/Sources/Domestic/DomesticStockPriceResult.swift
+++ b/KissMe/Sources/Domestic/DomesticStockPriceResult.swift
@@ -499,5 +499,16 @@ public struct MinutePriceResult: Codable {
            case lowestStockPrice = "stck_lwpr"
            case conclusionVolume = "cntg_vol"
        }
+        
+        public init(array: [String]) {
+            self.stockBusinessDate = array[0]
+            self.stockConclusionTime = array[1]
+            self.accumulatedTradingAmount = array[2]
+            self.currentStockPrice = array[3]
+            self.secondStockPrice = array[4]
+            self.highestStockPrice = array[5]
+            self.lowestStockPrice = array[6]
+            self.conclusionVolume = array[7]
+        }
    }
 }
--- a/KissMe/Sources/Domestic/Shop/DomesticShopProduct.swift
+++ b/KissMe/Sources/Domestic/Shop/DomesticShopProduct.swift
@@ -9,6 +9,7 @@ import Foundation


 public struct DomesticShop {
+    public typealias Product = ProductResponse.Item
 }


@@ -64,8 +65,6 @@ extension DomesticShop {
 extension DomesticShop {
    
    
-    public typealias Product = ProductResponse.Item
-    
    public struct ProductResponse: Codable {
        public let response: Response
        
@@ -124,7 +123,7 @@ extension DomesticShop {
                case corporationNo = "crno"
            }
            
-            public init(_ array: [String]) {
+            public init(array: [String]) {
                self.baseDate = array[0]
                
                /// shortCode 단축코드 명에 A000000 형태로 A문자가 붙어 있다.
--- a/KissMeConsole/Sources/KissConsole+CSV.swift
+++ b/KissMeConsole/Sources/KissConsole+CSV.swift
@@ -11,9 +11,57 @@ import KissMe

 extension KissConsole {
    
-    func writeShop(_ shopItems: [DomesticShop.Product], fileUrl: URL) {
+    var shopProductsUrl: URL {
+        URL.currentDirectory().appending(path: "data/shop-products.csv")
+    }
+    
+    func loadShop(_ profile: Bool = false) {
+        let appTime1 = Date.appTime
+        guard let stringCsv = try? String(contentsOfFile: shopProductsUrl.path) else {
+            return
+        }
+
+        let appTime2 = Date.appTime
+        if profile {
+            print("\tloading file \(appTime2 - appTime1) elapsed")
+        }
+        
+        var products = [String: [DomesticShop.Product]]()
+        let rows = stringCsv.split(separator: "\n")
+        
+        let appTime3 = Date.appTime
+        if profile {
+            print("\trow split \(appTime3 - appTime2) elapsed")
+        }
+        
+        for (i, row) in rows.enumerated() {
+            let array = row.split(separator: ",").map { String($0) }
+            if i == 0, array[0] == "baseDate" {
+                continue
+            }
+            
+            let product = DomesticShop.Product(array: array)
+            if var value = products[product.itemName] {
+                value.append(product)
+                products.updateValue(value, forKey: product.itemName)
+            }
+            else {
+                products[product.itemName] = [product]
+            }
+        }
+        let appTime4 = Date.appTime
+        if profile {
+            print("\tparse product \(appTime4 - appTime3) elapsed")
+        }
+        
+        setProducts(products)
+        let totalCount = products.reduce(0, { $0 + $1.value.count })
+        print("load products \(totalCount) with \(products.count) key")
+    }
+    
+    static func writeShop(_ shopItems: [DomesticShop.Product], fileUrl: URL) {
        var stringCsv: String = ""
-        let header = "baseDate,shortCode,isinCode,marketCategory,itemName,corporationNo,\n"
+        let header = "baseDate,shortCode,isinCode,marketCategory,itemName,corporationNo\n"
        stringCsv.append(header)
        for item in shopItems {
            let stringItem = [item.baseDate,
@@ -34,8 +82,8 @@ extension KissConsole {
        }
    }
    
-    func writeCandle(_ prices: [MinutePriceResult.OutputPrice], fileUrl: URL) {
-        var stringCsv: String = "stockBusinessDate,stockConclusionTime,accumulatedTradingAmount,currentStockPrice,secondStockPrice,highestStockPrice,lowestStockPrice,conclusionVolume,\n"
+    static func writeCandle(_ prices: [Domestic.Candle], fileUrl: URL) {
+        var stringCsv: String = "stockBusinessDate,stockConclusionTime,accumulatedTradingAmount,currentStockPrice,secondStockPrice,highestStockPrice,lowestStockPrice,conclusionVolume\n"
        let header = ""
        stringCsv.append(header)
        for item in prices {
@@ -57,4 +105,80 @@ extension KissConsole {
            print("\(error)")
        }
    }
+    
+    
+    enum CandleValidation: CustomStringConvertible {
+        case ok
+        case invalidFileName
+        case cannotRead
+        case invalidCsvHeader
+        case invalidBusinessDate
+        case invalidConclusionTime
+        
+        var description: String {
+            switch self {
+            case .ok: return "ok"
+            case .invalidFileName: return "invalidFileName"
+            case .cannotRead: return "cannotRead"
+            case .invalidCsvHeader: return "invalidCsvHeader"
+            case .invalidBusinessDate: return "invalidBusinessDate"
+            case .invalidConclusionTime: return "invalidConclusionTime"
+            }
+        }
+    }
+    
+    static func validateCandle(_ fileUrl: URL) -> CandleValidation {
+        let fileNameFrag = fileUrl.lastPathComponent.split(separator: ".")
+        guard fileNameFrag.count == 2 else {
+            return .invalidFileName
+        }
+        guard fileNameFrag[0].prefix(7) == "candle-", fileNameFrag[1] == "csv" else {
+            return .invalidFileName
+        }
+        
+        let fileDate = fileNameFrag[0].suffix(fileNameFrag[0].count - 7)
+        guard let stringCsv = try? String(contentsOfFile: fileUrl.path) else {
+            return .cannotRead
+        }
+        
+        var candles = [Domestic.Candle]()
+        let rows = stringCsv.split(separator: "\n")
+        for (i, row) in rows.enumerated() {
+            let array = row.split(separator: ",").map { String($0) }
+            if i == 0 {
+                if array.count != 8 {
+                    return .invalidCsvHeader
+                }
+                continue
+            }
+            
+            let candle = Domestic.Candle(array: array)
+            candles.append(candle)
+        }
+        
+        var curHH = 9, curMM = 0
+        for candle in candles.reversed() {
+            if candle.stockBusinessDate != fileDate {
+                return .invalidBusinessDate
+            }
+            guard let (hh, mm, _) = candle.stockConclusionTime.HHmmss else {
+                return .invalidConclusionTime
+            }
+            guard hh == curHH, mm == curMM else {
+                return .invalidConclusionTime
+            }
+            
+            // Finished to check
+            if hh == 18, mm == 0 {
+                break
+            }
+            
+            curMM += 1
+            if curMM >= 60 {
+                curMM = 0
+                curHH += 1
+            }
+        }
+        return .ok
+    }
 }
--- a/KissMeConsole/Sources/KissConsole.swift
+++ b/KissMeConsole/Sources/KissConsole.swift
@@ -205,7 +205,7 @@ extension KissConsole {
        try? FileManager.default.createDirectory(at: subPath, withIntermediateDirectories: true)
    }
    
-    private func setProducts(_ products: [String: [DomesticShop.Product]]) {
+    func setProducts(_ products: [String: [DomesticShop.Product]]) {
        productsLock.lock()
        self.products = products
        productsLock.unlock()
@@ -251,50 +251,6 @@ extension KissConsole {
        return all
    }
    
-    private var shopProductsUrl: URL {
-        URL.currentDirectory().appending(path: "data/shop-products.csv")
-    }
-    
-    private func loadShop(_ profile: Bool = false) {
-        let appTime1 = Date.appTime
-        guard let stringCsv = try? String(contentsOfFile: shopProductsUrl.path) else {
-            return
-        }
-
-        let appTime2 = Date.appTime
-        if profile {
-            print("\tloading file \(appTime2 - appTime1) elapsed")
-        }
-        
-        var products = [String: [DomesticShop.Product]]()
-        let rows = stringCsv.split(separator: "\n")
-        
-        let appTime3 = Date.appTime
-        if profile {
-            print("\trow split \(appTime3 - appTime2) elapsed")
-        }
-        
-        for row in rows {
-            let array = row.split(separator: ",").map { String($0) }
-            let product = DomesticShop.Product(array)
-            if var value = products[product.itemName] {
-                value.append(product)
-                products.updateValue(value, forKey: product.itemName)
-            }
-            else {
-                products[product.itemName] = [product]
-            }
-        }
-        let appTime4 = Date.appTime
-        if profile {
-            print("\tparse product \(appTime4 - appTime3) elapsed")
-        }
-        
-        setProducts(products)
-        let totalCount = products.reduce(0, { $0 + $1.value.count })
-        print("load products \(totalCount) with \(products.count) key")
-    }
-    
    private func lastLogin() {
        let profile = KissProfile()
        guard let isMock = profile.isMock else {
@@ -627,7 +583,7 @@ extension KissConsole {
            //nextTime.change(year: 2023, month: 5, day: 26)
            //nextTime.change(hour: 9, min: 1, sec: 0)
            
-            var candles = [MinutePriceResult.OutputPrice]()
+            var candles = [Domestic.Candle]()
            var count = 0
            
            while true {
@@ -673,7 +629,7 @@ extension KissConsole {
            let subFile = "\(subPath)/candle-\(minTime).csv"
            let fileUrl = URL.currentDirectory().appending(path: subFile)
            createSubpath(subPath)
-            writeCandle(candles, fileUrl: fileUrl)
+            KissConsole.writeCandle(candles, fileUrl: fileUrl)
            return true
        } catch {
            print("\(error)")
@@ -711,7 +667,7 @@ extension KissConsole {
            }
        }
        
-        writeShop(shopItems, fileUrl: shopProductsUrl)
+        KissConsole.writeShop(shopItems, fileUrl: shopProductsUrl)
    }
    
    private func getAllProduct(baseDate: Date) async -> [DomesticShop.Product] {
--- a/KissMeConsole/Sources/test.swift
+++ b/KissMeConsole/Sources/test.swift
@@ -100,3 +100,62 @@ private func test_xml_result() {
    }
     */
 }
+
+
+func subPathFiles(_ subpath: String) -> FileManager.DirectoryEnumerator? {
+    let baseUrl = URL.currentDirectory().appending(path: subpath)
+    let manager = FileManager.default
+    let resourceKeys : [URLResourceKey] = []
+    let enumerator = manager.enumerator(at: baseUrl, includingPropertiesForKeys: resourceKeys, options: [.skipsHiddenFiles]) { (url, error) -> Bool in
+        print("directoryEnumerator error at \(url): ", error)
+        return true
+    }
+    return enumerator
+}
+
+
+private func fix_first_csv_header_field() {
+    guard let enumerator = subPathFiles("data") else {
+        return
+    }
+    for case let fileUrl as URL in enumerator {
+        guard fileUrl.pathExtension == "csv" else {
+            continue
+        }
+        guard var stringCsv = try? String(contentsOfFile: fileUrl.path) else {
+            print("Cannot load \(fileUrl)")
+            continue
+        }
+        
+        guard let range = stringCsv.range(of: ",\n") else {
+            continue
+        }
+        stringCsv.remove(at: range.lowerBound)
+        
+        do {
+            try stringCsv.write(toFile: fileUrl.path, atomically: true, encoding: .utf8)
+            print("wrote \(fileUrl.lastPathComponent)")
+        } catch {
+            print(error)
+        }
+    }
+}
+
+
+private func check_candle_csv() {
+    guard let enumerator = subPathFiles("data") else {
+        return
+    }
+
+    for case let fileUrl as URL in enumerator {
+        let r = KissConsole.validateCandle(fileUrl)
+        switch r {
+        case .ok, .invalidFileName:
+            print("OK \(fileUrl)")
+            continue
+        default:
+            assertionFailure()
+            print("\(r) at \(fileUrl)")
+        }
+    }
+}
--- a/bin/data
+++ b/bin/data