You cannot control the File Size limit at Document library level OOTB. What you need, you have to write your own code may be some event receiver.
Check this code from this post
{
long validFileSize;
long currentFileSize;
if (properties.ListItem == null)
{
using (SPWeb web = properties.OpenWeb())
{
if (!web.CurrentUser.IsSiteAdmin)
{
if (properties.ListTitle.ToLower() == "images")
{
if (ConfigurationManager.AppSettings["FileSize"] != null)
{
validFileSize = long.Parse(ConfigurationManager.AppSettings["FileSize"].ToString());
}
else
{
validFileSize = 1000000;//1MB
}
currentFileSize = long.Parse(properties.AfterProperties["vti_filesize"].ToString());
if (currentFileSize > validFileSize)
{
return false;
}
}
}
}
}
else if (properties.ListItem.ParentList.Title.ToLower() == "images")
{
if (!properties.ListItem.Web.CurrentUser.IsSiteAdmin)
{
if (ConfigurationManager.AppSettings["FileSize"] != null)
{
validFileSize = long.Parse(ConfigurationManager.AppSettings["FileSize"].ToString());
}
else
{
validFileSize = 1000000;//1MB
}
currentFileSize = properties.ListItem.File.TotalLength;
if (currentFileSize > validFileSize)
{
return false;
}
}
}
return true;
}