NutzCN Logo
问答 关于大数据并发的问题
发布于 3033天前 作者 qq_0f18c1df 2146 次浏览 复制 上一个帖子 下一个帖子
标签:

哪位对数据库连接池配置比较熟悉的,场景是股市的行情入库,大概没5秒钟就会做一次10000行左右的update,每分钟会做一次10000行的插入,用的db2,刚开始还算可以1秒钟左右就能完成,慢慢的越来越慢,最慢的要几百秒,用了事务处理了。任务用的quartz。

关于dao.js的配置

相关的代码片段:

/**
	 * 
	 * initSZIndexQuote(更新SZIndexQuote表)
	 *
	 * @Title: initSZIndexQuote @Description: TODO @param @param bean
	 *         设定文件 @return void 返回类型 @throws
	 */
	public void updateSZIndexStockQuote(final SzSourceBean bean) {
		Trans.exec(new Atom() {
			public void run() {
				try {
					if (!(Lang.isEmpty(bean.getMd001s()))) {
						long a = System.currentTimeMillis();

						int i = 0;
						Sql sql = Sqls.create("update ADMINISTRATOR.SZ_INDEX_QUOTE set " + "PREVPRICE=@PREVPRICE,"
								+ "OPENPRICE=@OPENPRICE," + "LASTPRICE=@LASTPRICE," + "TIPTOPPRICE=@TIPTOPPRICE,"
								+ "LOWESTPRICE=@LOWESTPRICE," + "CURRENTAMOUNT=@CURRENTAMOUNT,"
								+ "TOTALAMOUNT=@TOTALAMOUNT," + "TOTALMONEY=@TOTALMONEY" + " where SECUCODE=@SECUCODE");
						for (SzSourceMd001 info : bean.getMd001s()) {
							sql.params().set("PREVPRICE", info.getPreClosePx()).set("OPENPRICE", info.getOpenPrice())
									.set("LASTPRICE", info.getTradePrice()).set("TIPTOPPRICE", info.getHighPrice())
									.set("LOWESTPRICE", info.getLowPrice()).set("CURRENTAMOUNT", info.getCurrentVol())
									.set("TOTALAMOUNT", info.getTradeVolume())
									.set("TOTALMONEY", info.getTotalValueTraded())
									.set("SECUCODE", info.getSecurityID());
							sql.addBatch();
							i++;
						}

						dao2().execute(sql);

						String dateStr = "SZ-" + DatesUtil.getFormatDate(bean.getMDTime());
						log(Constants.LOG_PARSE, "解析深证 [" + dateStr + "] :更新 [SZ_INDEX_QUOTE] 表数据 [" + i
								+ "] 条  执行时间  [" + ((System.currentTimeMillis() - a) / 1000f) + "] 秒");
					}

					if (!(Lang.isEmpty(bean.getMd002s()))) {
						long a = System.currentTimeMillis();

						int i = 0;
						Sql sql = Sqls.create("update ADMINISTRATOR.SZ_STOCK_QUOTE set " + "PREVPRICE=@PREVPRICE,"
								+ "OPENPRICE=@OPENPRICE," + "LASTPRICE=@LASTPRICE," + "TIPTOPPRICE=@TIPTOPPRICE,"
								+ "LOWESTPRICE=@LOWESTPRICE," + "CURRENTAMOUNT=@CURRENTAMOUNT,"
								+ "TOTALAMOUNT=@TOTALAMOUNT," + "TOTALMONEY=@TOTALMONEY," + "ASKPRICE5=@ASKPRICE5,"
								+ "ASKPRICE4=@ASKPRICE4," + "ASKPRICE3=@ASKPRICE3," + "ASKPRICE2=@ASKPRICE2,"
								+ "ASKPRICE1=@ASKPRICE1," + "BIDPRICE5=@BIDPRICE5," + "BIDPRICE4=@BIDPRICE4,"
								+ "BIDPRICE3=@BIDPRICE3," + "BIDPRICE2=@BIDPRICE2," + "BIDPRICE1=@BIDPRICE1,"
								+ "ASKAMOUNT5=@ASKAMOUNT5," + "ASKAMOUNT4=@ASKAMOUNT4," + "ASKAMOUNT3=@ASKAMOUNT3,"
								+ "ASKAMOUNT2=@ASKAMOUNT2," + "ASKAMOUNT1=@ASKAMOUNT1," + "BIDAMOUNT5=@BIDAMOUNT5,"
								+ "BIDAMOUNT4=@BIDAMOUNT4," + "BIDAMOUNT3=@BIDAMOUNT3," + "BIDAMOUNT2=@BIDAMOUNT2,"
								+ "BIDAMOUNT1=@BIDAMOUNT1" + " where SECUCODE=@SECUCODE");

						for (SzSourceMd002 info : bean.getMd002s()) {
							sql.params().set("PREVPRICE", info.getPreClosePx()).set("OPENPRICE", info.getOpenPrice())
									.set("LASTPRICE", info.getTradePrice()).set("TIPTOPPRICE", info.getHighPrice())
									.set("LOWESTPRICE", info.getLowPrice()).set("CURRENTAMOUNT", info.getCurrentVol())
									.set("TOTALAMOUNT", info.getTradeVolume())
									.set("TOTALMONEY", info.getTotalValueTraded()).set("SECUCODE", info.getSecurityID())
									.set("BIDPRICE5", info.getBuyPrice5()).set("BIDPRICE4", info.getBuyPrice4())
									.set("BIDPRICE3", info.getBuyPrice3()).set("BIDPRICE2", info.getBuyPrice2())
									.set("BIDPRICE1", info.getBuyPrice1()).set("ASKPRICE5", info.getSellPrice5())
									.set("ASKPRICE4", info.getSellPrice4()).set("ASKPRICE3", info.getSellPrice3())
									.set("ASKPRICE2", info.getSellPrice2()).set("ASKPRICE1", info.getSellPrice1())
									.set("BIDAMOUNT1", info.getBuyVolume1()).set("BIDAMOUNT2", info.getBuyVolume2())
									.set("BIDAMOUNT3", info.getBuyVolume3()).set("BIDAMOUNT4", info.getBuyVolume4())
									.set("BIDAMOUNT5", info.getBuyVolume5()).set("ASKAMOUNT1", info.getSellVolume1())
									.set("ASKAMOUNT2", info.getSellVolume2()).set("ASKAMOUNT3", info.getSellVolume3())
									.set("ASKAMOUNT4", info.getSellVolume4()).set("ASKAMOUNT5", info.getSellVolume5());
							sql.addBatch();
							i++;
						}
						dao2().execute(sql);

						String dateStr = "SZ-" + DatesUtil.getFormatDate(bean.getMDTime());
						log(Constants.LOG_PARSE, "解析深证 [" + dateStr + "] :更新 [SZ_STOCK_QUOTE] 表数据 [" + i
								+ "] 条  执行时间  [" + ((System.currentTimeMillis() - a) / 1000f) + "] 秒");
					}
				} catch (Exception e) {
					log.error(e);
				}
			}
		});
	}
dataSource2 : {
		type : "com.alibaba.druid.pool.DruidDataSource",
		events : {
			create : "init",
			depose : 'close'
		},
		fields : {
			driverClassName : {
				java : "$conf.get('db2.driverClassName')"
		
			},
			url : {
				java : "$conf.get('db2.url')"
			},
			username : {
				java : "$conf.get('db2.username')"
			},
			password : {
				java : "$conf.get('db2.password')"
			},
			initialSize:20,
			minIdle:20,
			maxActive : 200,
			timeBetweenEvictionRunsMillis:1800000,
            maxWait: 15000, // 若不配置此项,如果数据库未启动,druid会一直等可用连接,卡住启动过程,
            defaultAutoCommit : true, // 提高fastInsert的性能
            poolPreparedStatements: true,
            maxPoolPreparedStatementPerConnectionSize : 100,
            testWhileIdle : true,
            validationQuery : {
            	java : "$conf.get('db2.validationQuery')"
            }
		}
	},
4 回复

一次更新一万条是不是太多了,锁表了导致wait?

dao.execute之前也计时一下

题外话

defaultAutoCommit:false才能提高非事务操作下的fastInsert性能

来自炫酷的 NutzCN

好的,我再测试下

@wendal 我是事务操作的,defaultAutoCommit:false有效吗

@qq_0f18c1df 事务下总是defaultAutoCommit=false, 所以没影响

添加回复
请先登陆
回到顶部